{"what":"Measurements that chose or rejected a form, none of them shipped: fp32 attention in an fp16 decoder (38 runs), S = 128 (20 runs), the fixed-grid price with the author's code alone (native vs 256 / 448 / 672 / 896), the lm_head table format (fp32 / fp16 / int8), the residual-stream absmax over every fixture run, and the round-3 check of the fp16 graph's low-cosine rows against fp16 torch.","sources":[{"file":"results/readout_fp16attn32_pf64.json","bytes":194438,"sha256":"c19727d9487b50a658d0b15eba40f7fa6a61ba2c9219a10a9beb32bc0c8d51b8"},{"file":"logs/r4_export_fp16attn32_pf64.json","bytes":8951,"sha256":"e9eb8b44d9ebc1761c0f927daee0a9b0eb63b60aae5cfe9557a0c29ee9e86628"},{"file":"results/r4_runs_fp16attn32.json","bytes":1865,"sha256":"420ea2cf1ee929f87cffa74c5c426cb68fef6077a46ebe22125781926f9c80f0"},{"file":"results/readout_fp16_pf128.json","bytes":108468,"sha256":"61351e164c3028d6f0677c595fa874c6b37d89a5085d4c7389ce362366652353"},{"file":"logs/r4_export_fp16_pf128.json","bytes":6852,"sha256":"0d734466ace53072c8ed9733390142f7df186132e2eb1c880593e261c8888eda"},{"file":"results/r4_runs_s128.json","bytes":974,"sha256":"fdc9235cbefef4181b266d8a704a2cdb235545e72e66cc392414d9664447a0d9"},{"file":"results/grid_price_large.json","bytes":62634,"sha256":"dbd8606936fcb3e2e5274dd290cf334d16af13cca46bbe7e60073f5ccf469935"},{"file":"results/lm_head_table.json","bytes":2835,"sha256":"4da298b2e8b31b35f3e45b349555b61560e12c566c85e0987f98da354611fe2f"},{"file":"results/lm_head_table_effect.json","bytes":1185,"sha256":"c58b1fea3b5a2b810fc65fddd5794727905ce66eb0f00247cba5514ad064565f"},{"file":"results/activation_absmax.json","bytes":15684,"sha256":"5ccccd1a448c9e15dc5f0dc36bc5d359c8cbec214e0797a6962004e91580365d"},{"file":"results/r3_fp16_torch_crosscheck.json","bytes":9726,"sha256":"50e307d6d34864eaa75be52a154a3a3a9966a1fbb9ebff6d2c95ad73edf091eb"}],"trimmed":{"removed":["fp16attn32 / pf128 readouts: runs[].call_ms (per-call ms; the timing block keeps the medians)","fp16attn32 / pf128 readouts: runs[].npz / hidden_npz / npz_key (lane file names)","fp16attn32 / pf128 readouts: runs[].hidden.lowest_positions (each run keeps min_pos_cos, its index and the counts below 0.99 / 0.9 / 0.5)","fp16attn32 / pf128 readouts: runs[].questions[].probs_oracle (the oracle's probabilities are in fixtures-clef-flash.json)","fp16attn32 / pf128 readouts: oracle.npz_sha256 (sha256 of the lane's per-run oracle npz files)","fp16attn32 / pf128 readouts: processes[].npz / npz_sha256 / descriptor (the descriptor is the top-level one: descriptor_same_in_every_process)","fp16attn32 / pf128 readouts: compare_with.rows (per-run comparison rows; the compare_with summary stays)"],"serialization":"compact json (no indentation)","full_copies":"the source files above, in the lane directory $ZOO_WORK_ROOT/_clefflash/ (not in the repository)"},"fp16attn32_pf64":{"runs_file":{"runs":[["own_t01","text"],["own_t04","text"],["own_t14","text"],["own_t15","text"],["own_t16","text"],["own_j08","text"],["own_j12","text"],["img_03","native"],["img_04","g448"],["img_04","native"],["img_06","g448"],["img_07","native"],["img_08","native"],["img_09","native"],["photo_01","g448"],["photo_02","g448"],["photo_02","native"],["photo_03","g256"],["photo_03","native"],["photo_04","g448"],["photo_04","native"],["semif_9473603b44af63beab79","text"],["semif_fdb9fc214aea2710763a","text"],["semif_97a35499bd8717f3e786","text"],["semif_599b3092b6fd18312d23","text"],["semif_f3d13cf184f916677def","text"],["semif_0aa1447680e4f25862bb","text"],["semif_6d64a8459f64eac7f4cc","text"],["semif_2708193212d8a4a523c7","text"],["own_j01","text"],["img_01","g256"],["semif_a3f18f3a63d45345942b","text"],["semif_f40beba9088c8db8bbd6","text"],["semif_0b43ea8e24e74d621f1b","text"],["semif_942ec65dac0bfb9874f9","text"],["semif_e0c140e9222d1bce5647","text"],["semif_8e8c3804a3c15ebb31e3","text"],["semif_56c419b12cd37cbe790c","text"]],"why":"round 4 step 4 (fp16attn32 diagnostic): the 29 runs with a position cosine < 0.99 in readout_fp16_pf16.json (results/r4_poscos_fp16_pf16.json) + own_t01/text, own_j01/text, img_01/g256, img_04/g448 + the first 7 SemIf records (launch 2026-10-03-A-clef-flash-coreai-opus-r4.md step 4)"},"export_record":{"mode":"fp16","name":"clef_flash_decode_fp16attn32_pf64","hf_id":"Cloudflare/clef-flash","revision":"17f0b0ad64efb65d273590632833508766b2aae6","prefill_chunk":64,"argv":["fp16","--variant","fp16attn32","--prefill-chunk","64","--aot","--record","<lane>/logs/r4_export_fp16attn32_pf64.json"],"pid":43834,"started":"2026-10-03T13:06:29+09:00","script_sha256":"fbc1d712842357bd57735f9acf7d47042f14f966d7c4ca5c5a358db3cfbd5a68","module_sha256":"6342008b9fde0a521aa4db5b7ac9e164c93086a7cae92ee19ba1c2e356f59b8f","parent_sha256":"7f7fd56967f7ba4ca9e9ca12092205df638d6996470454cfc0845b495090211f","export":{"bundle":"<lane>/exports/bundles/clef_flash_decode_fp16attn32_pf64","name":"clef_flash_decode_fp16attn32_pf64","aimodel":"<lane>/exports/bundles/clef_flash_decode_fp16attn32_pf64/clef_flash_decode_fp16attn32_pf64.aimodel","load_report":{"checkpoint_text_keys":427,"checkpoint_other_keys_skipped":455,"module_tensors":426,"unread_checkpoint_keys":[],"module_tensors_not_in_checkpoint":[],"tied_lm_head":false,"intentionally_unread_keys":["lm_head.weight"],"lm_head_untied":true,"checkpoint_lm_head":{"file":"model-00001-of-00004.safetensors","dtype":"BF16","shape":[248320,4096]},"module_has_lm_head_weight":false},"loopfree_unrolled_linear_layers":24,"trace_kv_len":2048,"max_ctx":4096,"n_image_max":1024,"functions":["main"],"query_len":64,"gdn_scan":"unroll","output_names":["hidden"],"quantization":null,"variant":{"name":"fp16attn32","fp32_compute_modules":["model.layers.3.self_attn.q_proj","model.layers.3.self_attn.k_proj","model.layers.3.self_attn.v_proj","model.layers.3.self_attn.o_proj","model.layers.3.self_attn.sdpa","model.layers.7.self_attn.q_proj","model.layers.7.self_attn.k_proj","model.layers.7.self_attn.v_proj","model.layers.7.self_attn.o_proj","model.layers.7.self_attn.sdpa","model.layers.11.self_attn.q_proj","model.layers.11.self_attn.k_proj","model.layers.11.self_attn.v_proj","model.layers.11.self_attn.o_proj","model.layers.11.self_attn.sdpa","model.layers.15.self_attn.q_proj","model.layers.15.self_attn.k_proj","model.layers.15.self_attn.v_proj","model.layers.15.self_attn.o_proj","model.layers.15.self_attn.sdpa","model.layers.19.self_attn.q_proj","model.layers.19.self_attn.k_proj","model.layers.19.self_attn.v_proj","model.layers.19.self_attn.o_proj","model.layers.19.self_attn.sdpa","model.layers.23.self_attn.q_proj","model.layers.23.self_attn.k_proj","model.layers.23.self_attn.v_proj","model.layers.23.self_attn.o_proj","model.layers.23.self_attn.sdpa","model.layers.27.self_attn.q_proj","model.layers.27.self_attn.k_proj","model.layers.27.self_attn.v_proj","model.layers.27.self_attn.o_proj","model.layers.27.self_attn.sdpa","model.layers.31.self_attn.q_proj","model.layers.31.self_attn.k_proj","model.layers.31.self_attn.v_proj","model.layers.31.self_attn.o_proj","model.layers.31.self_attn.sdpa"],"how":"every full-attention layer's q_proj / k_proj / v_proj / o_proj and its SDPA run in fp32: the linears keep fp16 weights with an fp32 view (export_qwen38vl_pipelined.fp16_storage_fp32_compute), each wrapped module casts its tensor inputs to fp32 and its output back to fp16; the residual stream, the norms, RoPE and the KV cache stay fp16 (the SDPA reads the fp16 cache, cast up)"},"seconds":{"load":5.644262499990873,"quantize":0.08288395800627768,"export":101.42803845898015,"optimize":1.14101487500011,"export_optimize":102.56905333398026,"save":11.537365458003478,"total":119.92996162499185},"du_aimodel":"15G","du_bundle":"15G","aimodel_files":{"main.hash":32,"main.mlirb":15880279555,"metadata.json":105},"main_mlirb":{"bytes":15880279555,"sha256":"69e07d486617c82ab036ef0fece51a4b8a2d4ed69ac83ac05ec71a84f348f37f"},"metadata_sha256":"1a67faced128b2c02161fe938607f8b46ba06f5b1656c3d40169c51664b36aca","kind":"decision-backbone","tokenizer_sha256":{"chat_template.jinja":"a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715","tokenizer.json":"06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523","tokenizer_config.json":"91a08f825d370d085d692e04cf117cdd7faad7bf18e996f1e6031b6dab03db72"},"tokenizer_verbatim_from_snapshot":true,"disk":{"before_load":{"time":"2026-10-03T13:06:31+09:00","free_bytes":174861778944,"free_gib":162.9,"caches":{"coreai_cache_python":"32G","mpsgraph_scratch":{"/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph":"0B"}}},"after_load":{"time":"2026-10-03T13:06:37+09:00","free_bytes":174861783040,"free_gib":162.9,"caches":{"coreai_cache_python":"32G","mpsgraph_scratch":{"/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph":"0B"}}},"after_export":{"time":"2026-10-03T13:08:19+09:00","free_bytes":162669740032,"free_gib":151.5,"caches":{"coreai_cache_python":"32G","mpsgraph_scratch":{"/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph":"0B"}}},"after_save":{"time":"2026-10-03T13:08:31+09:00","free_bytes":144784031744,"free_gib":134.8,"caches":{"coreai_cache_python":"32G","mpsgraph_scratch":{"/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph":"0B"}}}}},"aot":{"aimodelc":"<lane>/exports/bundles_aotc/clef_flash_decode_fp16attn32_pf64.h16c.aimodelc","flags":["--platform","macOS","--preferred-compute","gpu","--architecture","h16c","--expect-frequent-reshapes"],"seconds":108.98382550000679,"coreai_build":"/var/run/com.apple.security.cryptexd/mnt/com.apple.MobileAsset.MetalToolchain-v27.1.266.1.6t2kcX/Metal.xctoolchain/usr/bin/coreai-build","command":["/var/run/com.apple.security.cryptexd/mnt/com.apple.MobileAsset.MetalToolchain-v27.1.266.1.6t2kcX/Metal.xctoolchain/usr/bin/coreai-build","compile","<lane>/exports/bundles/clef_flash_decode_fp16attn32_pf64/clef_flash_decode_fp16attn32_pf64.aimodel","--output","<lane>/exports/bundles_aotc","--platform","macOS","--preferred-compute","gpu","--architecture","h16c","--expect-frequent-reshapes"],"disk":{"before":{"time":"2026-10-03T13:08:36+09:00","free_bytes":144763883520,"free_gib":134.8,"caches":{"coreai_cache_python":"32G","mpsgraph_scratch":{"/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph":"0B"}}},"after":{"time":"2026-10-03T13:10:25+09:00","free_bytes":111851511808,"free_gib":104.2,"caches":{"coreai_cache_python":"32G","mpsgraph_scratch":{"/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph":"0B"}}}},"du_aimodelc":"27G","digest":{"bytes":28794735683,"tree_sha256":"ad4f44eecb7a346472a3cf729c432354bf469638b45c1207ec79efcc4ce2e862","files":{"main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/manifest.plist":"61acf2f483fc9d862be5ad15ce917d3f0f9e6421fe7dbd88f46c232b67f9206b","main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/original_model_0.mpsgraph":"3dcd7977f14d58375d10f05772ba6c64befe1f5cc958afd09214aa880a65259d","main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/resources.bin":"3a7ecf8a316ca5bf0daaacb190ce1b7656faf3a3001ba13e3b79126e80f18fb2","main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/specialized_model_1.mpsgraph":"a6e6058a019d514ddb9a289fca3f551c1da6c0f5bad578f36a720c407ae9ad0e","main-h16c.mlirb":"47f82b9f37d353b7df4577a3f1fd2a8b45e605778b409092ff7649d9a37b5346","main.hash":"99a651d132d8f88dc030eed5770ef1e2c7f836ec19fb9c4fa895c22ad9d252a4","metadata.json":"2b84fe051ad9f4c3cccfb2408a0d31b14ceef8bb920381545613ad737767d238","stats.json":"9cad1c0872966ab5ef13d000e88be6ae75e7b1fa153c0e21ae6be45de68d1177"}}},"finished":"2026-10-03T13:10:35+09:00"},"readout":{"schema":"clef-flash-decoder-readout-gate/1","gate":"decoder alone on the Mac GPU (oracle ids, oracle fp32 image_embeds -> fp16, zero-padded to 1024 rows), the fp16 hidden read through the author's fp32 head","bundle":{"bundle":"<lane>/exports/bundles/clef_flash_decode_fp16attn32_pf64","name":"clef_flash_decode_fp16attn32_pf64","kind":"decision-backbone","compression":null,"prefill_chunk":64,"metadata_sha256":"1a67faced128b2c02161fe938607f8b46ba06f5b1656c3d40169c51664b36aca","main_mlirb":{"bytes":15880279555,"sha256":"69e07d486617c82ab036ef0fece51a4b8a2d4ed69ac83ac05ec71a84f348f37f"},"tokenizer_sha256":{"chat_template.jinja":"a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715","tokenizer.json":"06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523","tokenizer_config.json":"91a08f825d370d085d692e04cf117cdd7faad7bf18e996f1e6031b6dab03db72"},"aimodelc":{"path":"<lane>/exports/bundles_aotc/clef_flash_decode_fp16attn32_pf64.h16c.aimodelc","bytes":28794735683,"tree_sha256":"ad4f44eecb7a346472a3cf729c432354bf469638b45c1207ec79efcc4ce2e862","files":{"main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/manifest.plist":"61acf2f483fc9d862be5ad15ce917d3f0f9e6421fe7dbd88f46c232b67f9206b","main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/original_model_0.mpsgraph":"3dcd7977f14d58375d10f05772ba6c64befe1f5cc958afd09214aa880a65259d","main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/resources.bin":"3a7ecf8a316ca5bf0daaacb190ce1b7656faf3a3001ba13e3b79126e80f18fb2","main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/specialized_model_1.mpsgraph":"a6e6058a019d514ddb9a289fca3f551c1da6c0f5bad578f36a720c407ae9ad0e","main-h16c.mlirb":"47f82b9f37d353b7df4577a3f1fd2a8b45e605778b409092ff7649d9a37b5346","main.hash":"99a651d132d8f88dc030eed5770ef1e2c7f836ec19fb9c4fa895c22ad9d252a4","metadata.json":"2b84fe051ad9f4c3cccfb2408a0d31b14ceef8bb920381545613ad737767d238","stats.json":"9cad1c0872966ab5ef13d000e88be6ae75e7b1fa153c0e21ae6be45de68d1177"}}},"chunk":64,"max_ctx":4096,"readout":"fresh zero states; a row of T ids runs as ceil(T / 64) calls of 'main' (static S = 64): call k gets ids[64k : 64k + 64] with position_ids 0..64k+63; the last call is padded with <|endoftext|> (248044) and the hidden rows of the padded positions are discarded (causal: they cannot reach a real position). The hidden rows [1, 64, 4096] of every call, concatenated and cut to T, are the backbone's final-norm last_hidden_state [T, 4096] the head reads.","descriptor":{"function":"main","inputs":{"input_ids":[[1,64],"int32"],"position_ids":[[1,-1],"int32"],"image_embeds":[[1024,4096],"float16"],"image_rc":[[1024,2],"int32"],"rope_shift_start":[[1],"int32"],"rope_shift_amount":[[1],"int32"]},"outputs":{"hidden":[[1,64,4096],"float16"]},"states":{"keyCache":[[8,1,4,-1,256],"float16"],"valueCache":[[8,1,4,-1,256],"float16"],"convState":[[24,1,8192,3],"float16"],"recState":[[24,1,32,128,128],"float16"]}},"descriptor_same_in_every_process":true,"contract":{"inputs":{"input_ids":[[1,64],"int32"],"position_ids":[[1,-1],"int32"],"image_embeds":[[1024,4096],"float16"],"image_rc":[[1024,2],"int32"],"rope_shift_start":[[1],"int32"],"rope_shift_amount":[[1],"int32"]},"outputs":{"hidden":[[1,64,4096],"float16"]},"states":{"keyCache":[[8,1,4,-1,256],"float16"],"valueCache":[[8,1,4,-1,256],"float16"],"convState":[[24,1,8192,3],"float16"],"recState":[[24,1,32,128,128],"float16"]}},"oracle":{"path":"<lane>/oracle/records_oracle.json","sha256":"d8adaa746a5d575e654212ca78f5634af334aaa62eb53ef150d5d5925b1fa7c0","source":"Cloudflare/clef-flash@17f0b0ad64efb65d273590632833508766b2aae6","versions":{"python":"3.12.11","torch":"2.9.0","torchvision":"0.24.0","transformers":"5.17.0","pillow":"12.3.0","numpy":"2.5.3","huggingface_hub":"1.33.0"}},"subset":"list","runs_file":{"path":"<lane>/results/r4_runs_fp16attn32.json","sha256":"420ea2cf1ee929f87cffa74c5c426cb68fef6077a46ebe22125781926f9c80f0","why":"round 4 step 4 (fp16attn32 diagnostic): the 29 runs with a position cosine < 0.99 in readout_fp16_pf16.json (results/r4_poscos_fp16_pf16.json) + own_t01/text, own_j01/text, img_01/g256, img_04/g448 + the first 7 SemIf records (launch 2026-10-03-A-clef-flash-coreai-opus-r4.md step 4)"},"script":{"path":"conversion/clef_flash/readout_gate.py","sha256":"43727fedef59375e6f4c77d3a32d9623659ea6f71ad8948bb439c06c11928006"},"skipped":[{"run":"photo_01/native","reason":"N = 40 x 30 = 1,200 image rows > the graph's 1024-row buffer (outside the fixed-grid path), not run","questions":4}],"bar":{"max_abs_dp":0.02,"mean_of_run_mean_abs_dp":0.002,"near_tie_top2_margin":0.02,"argmax":"every question with an oracle top-2 margin above 0.02; near-ties listed apart","max_abs_dp_applies_to":"every option of every question, near-ties included","reset":"bit-equal hidden rows in every process","mean_of_run_mean_abs_dp_definition":"mean over runs of the mean |dp| over all (question, option) entries of the run"},"environment":{"versions":{"python":"3.11.13","numpy":"2.3.5","coreai-core":"1.0.0b2","coreai-torch":"0.4.1","coreai-models":"0.1.0","torch":"2.9.0","safetensors":"0.7.0"},"platform":"macOS-27.0-arm64-arm-64bit","macos_build":"26A428","chip":"Apple M4 Max","runtime":"coreai python runtime, AOT h16c GPU .aimodelc, SpecializationOptions.default(), no JIT","head":"the author's JointSchemaHead, fp32, CPU (parity_decoder_torch.AuthorHead)","gpu":"shared with other sessions, no _GPU_LOCK held (times are contended reference values)"},"other_gpu_processes":{"note":"process listing removed before publication (4 lines); it showed other sessions' GPU work at the time = the times are contended"},"processes":[{"shard":"shard_00","runs":["own_t01/text/base","own_t04/text/base","own_t14/text/base","own_t15/text/base","own_t16/text/base","own_j08/text/base","own_j12/text/base","img_03/native/base","img_04/g448/base","img_04/native/base","img_06/g448/base","img_07/native/base","img_08/native/base","img_09/native/base","photo_01/g448/base","photo_02/g448/base","photo_02/native/base","photo_03/g256/base","photo_03/native/base","photo_04/g448/base","photo_04/native/base","semif_9473603b44af63beab79/text/base","semif_fdb9fc214aea2710763a/text/base","semif_97a35499bd8717f3e786/text/base","semif_599b3092b6fd18312d23/text/base","semif_f3d13cf184f916677def/text/base","semif_0aa1447680e4f25862bb/text/base","semif_6d64a8459f64eac7f4cc/text/base","semif_2708193212d8a4a523c7/text/base","own_j01/text/base","img_01/g256/base","semif_a3f18f3a63d45345942b/text/base","semif_f40beba9088c8db8bbd6/text/base","semif_0b43ea8e24e74d621f1b/text/base","semif_942ec65dac0bfb9874f9/text/base","semif_e0c140e9222d1bce5647/text/base","semif_8e8c3804a3c15ebb31e3/text/base","semif_56c419b12cd37cbe790c/text/base"],"prompts":39,"pid":46570,"load_seconds":23.777184999984456,"load_split_seconds":{"model":4.5597041659930255,"main":19.21748083399143},"function_names":["main"],"process_wall_seconds":109.94605995799066,"reset_check":{"run":["own_t01","text","base"],"bit_equal":true,"hidden_max_abs_diff":0.0,"wall_seconds":1.0144728750165086},"contract_mismatch":[]}],"summary":{"runs":38,"expected_runs":38,"missing_runs":[],"questions":104,"argmax_equal":104,"questions_non_near_tie":102,"argmax_equal_non_near_tie":102,"near_tie_questions":2,"argmax_equal_near_tie":2,"max_abs_dp":0.007514894008636475,"max_abs_dp_non_near_tie":0.007514894008636475,"mean_of_run_mean_abs_dp":0.00036387249536020674,"mean_question_max_abs_dp":0.00045022434026647644,"max_abs_dlogit":0.05959749221801758,"min_pos_cos":0.37844254882577144,"positions":24425,"positions_below":{"0.99":53,"0.9":5,"0.5":1},"positions_below_0.99_image_rows":35,"runs_with_positions_below_0.99":["own_t01/text","own_t04/text","own_t14/text","own_t15/text","own_t16/text","own_j08/text","img_03/native","img_04/g448","img_04/native","img_06/g448","img_07/native","img_08/native","img_09/native","photo_01/g448","photo_02/g448","photo_02/native","photo_03/g256","photo_03/native","photo_04/g448","photo_04/native","semif_9473603b44af63beab79/text","semif_fdb9fc214aea2710763a/text","semif_97a35499bd8717f3e786/text","semif_599b3092b6fd18312d23/text","semif_f3d13cf184f916677def/text","semif_0aa1447680e4f25862bb/text","semif_6d64a8459f64eac7f4cc/text","semif_2708193212d8a4a523c7/text"],"hidden_max_abs_diff":20.76478672027588,"hidden_max_rel_diff":0.4197689096594061,"worst_run":{"id":"semif_0aa1447680e4f25862bb","arm":"text","max_abs_dp":0.007514894008636475},"finite_all":true,"tokens":24425,"calls":396,"ms_per_call_median":143.5307704959996,"ms_per_call_warm_median":143.50587499211542},"by_source_arm":{"cc0_photo/g256":{"runs":1,"expected_runs":1,"missing_runs":[],"questions":4,"argmax_equal":4,"questions_non_near_tie":4,"argmax_equal_non_near_tie":4,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":5.698204040527344e-05,"max_abs_dp_non_near_tie":5.698204040527344e-05,"mean_of_run_mean_abs_dp":2.8507706398765247e-05,"mean_question_max_abs_dp":4.277750849723816e-05,"max_abs_dlogit":0.004570960998535156,"min_pos_cos":0.9700445583965338,"positions":505,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0,"runs_with_positions_below_0.99":["photo_03/g256"],"hidden_max_abs_diff":1.9747991561889648,"hidden_max_rel_diff":0.040871769441302615,"worst_run":{"id":"photo_03","arm":"g256","max_abs_dp":5.698204040527344e-05},"finite_all":true,"tokens":505,"calls":8,"ms_per_call_median":143.35002050211187,"ms_per_call_warm_median":143.35002050211187},"cc0_photo/g448":{"runs":3,"expected_runs":3,"missing_runs":[],"questions":12,"argmax_equal":12,"questions_non_near_tie":12,"argmax_equal_non_near_tie":12,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.004601955413818359,"max_abs_dp_non_near_tie":0.004601955413818359,"mean_of_run_mean_abs_dp":0.0003325177090935027,"mean_question_max_abs_dp":0.0006386179011315107,"max_abs_dlogit":0.015121936798095703,"min_pos_cos":0.37844254882577144,"positions":1953,"positions_below":{"0.99":6,"0.9":2,"0.5":1},"positions_below_0.99_image_rows":6,"runs_with_positions_below_0.99":["photo_01/g448","photo_02/g448","photo_04/g448"],"hidden_max_abs_diff":20.76478672027588,"hidden_max_rel_diff":0.4197689096594061,"worst_run":{"id":"photo_01","arm":"g448","max_abs_dp":0.004601955413818359},"finite_all":true,"tokens":1953,"calls":31,"ms_per_call_median":143.377042026259,"ms_per_call_warm_median":143.377042026259},"cc0_photo/native":{"runs":3,"expected_runs":3,"missing_runs":[],"questions":12,"argmax_equal":12,"questions_non_near_tie":12,"argmax_equal_non_near_tie":12,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.00020879507064819336,"max_abs_dp_non_near_tie":0.00020879507064819336,"mean_of_run_mean_abs_dp":6.620732979833061e-05,"mean_question_max_abs_dp":0.00010362830168257157,"max_abs_dlogit":0.011842012405395508,"min_pos_cos":0.6753120540250005,"positions":3114,"positions_below":{"0.99":12,"0.9":2,"0.5":0},"positions_below_0.99_image_rows":12,"runs_with_positions_below_0.99":["photo_02/native","photo_03/native","photo_04/native"],"hidden_max_abs_diff":16.05294418334961,"hidden_max_rel_diff":0.33224314213814116,"worst_run":{"id":"photo_03","arm":"native","max_abs_dp":0.00020879507064819336},"finite_all":true,"tokens":3114,"calls":51,"ms_per_call_median":143.7149590055924,"ms_per_call_warm_median":143.7149590055924},"own_image/g256":{"runs":1,"expected_runs":1,"missing_runs":[],"questions":3,"argmax_equal":3,"questions_non_near_tie":2,"argmax_equal_non_near_tie":2,"near_tie_questions":1,"argmax_equal_near_tie":1,"max_abs_dp":0.0015357136726379395,"max_abs_dp_non_near_tie":0.0015357136726379395,"mean_of_run_mean_abs_dp":0.00039574461565776306,"mean_question_max_abs_dp":0.0007255673408508301,"max_abs_dlogit":0.01240241527557373,"min_pos_cos":0.999530785693714,"positions":430,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0,"runs_with_positions_below_0.99":[],"hidden_max_abs_diff":0.6985244750976562,"hidden_max_rel_diff":0.014457131605421066,"worst_run":{"id":"img_01","arm":"g256","max_abs_dp":0.0015357136726379395},"finite_all":true,"tokens":430,"calls":7,"ms_per_call_median":143.45058298204094,"ms_per_call_warm_median":143.45058298204094},"own_image/g448":{"runs":2,"expected_runs":2,"missing_runs":[],"questions":7,"argmax_equal":7,"questions_non_near_tie":6,"argmax_equal_non_near_tie":6,"near_tie_questions":1,"argmax_equal_near_tie":1,"max_abs_dp":0.0018284618854522705,"max_abs_dp_non_near_tie":0.0018284618854522705,"mean_of_run_mean_abs_dp":0.0004232728388160467,"mean_question_max_abs_dp":0.0006313169641154153,"max_abs_dlogit":0.02442002296447754,"min_pos_cos":0.9766007468198847,"positions":1287,"positions_below":{"0.99":4,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":4,"runs_with_positions_below_0.99":["img_04/g448","img_06/g448"],"hidden_max_abs_diff":3.14107065461576,"hidden_max_rel_diff":0.06500970754010807,"worst_run":{"id":"img_06","arm":"g448","max_abs_dp":0.0018284618854522705},"finite_all":true,"tokens":1287,"calls":21,"ms_per_call_median":143.17150000715628,"ms_per_call_warm_median":143.17150000715628},"own_image/native":{"runs":5,"expected_runs":5,"missing_runs":[],"questions":18,"argmax_equal":18,"questions_non_near_tie":18,"argmax_equal_non_near_tie":18,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.0014417171478271484,"max_abs_dp_non_near_tie":0.0014417171478271484,"mean_of_run_mean_abs_dp":0.00013070140388749896,"mean_question_max_abs_dp":0.00024487787029809423,"max_abs_dlogit":0.01603877544403076,"min_pos_cos":0.6492394238482659,"positions":4347,"positions_below":{"0.99":13,"0.9":1,"0.5":0},"positions_below_0.99_image_rows":13,"runs_with_positions_below_0.99":["img_03/native","img_04/native","img_07/native","img_08/native","img_09/native"],"hidden_max_abs_diff":12.449501037597656,"hidden_max_rel_diff":0.2576637217161507,"worst_run":{"id":"img_08","arm":"native","max_abs_dp":0.0014417171478271484},"finite_all":true,"tokens":4347,"calls":70,"ms_per_call_median":143.4589584969217,"ms_per_call_warm_median":143.4589584969217},"own_json/text":{"runs":3,"expected_runs":3,"missing_runs":[],"questions":10,"argmax_equal":10,"questions_non_near_tie":10,"argmax_equal_non_near_tie":10,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.003338932991027832,"max_abs_dp_non_near_tie":0.003338932991027832,"mean_of_run_mean_abs_dp":0.0004814869007812097,"mean_question_max_abs_dp":0.0008105059154331684,"max_abs_dlogit":0.016298532485961914,"min_pos_cos":0.9584726391881166,"positions":1511,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0,"runs_with_positions_below_0.99":["own_j08/text"],"hidden_max_abs_diff":4.445657730102539,"hidden_max_rel_diff":0.09201049204743354,"worst_run":{"id":"own_j12","arm":"text","max_abs_dp":0.003338932991027832},"finite_all":true,"tokens":1511,"calls":25,"ms_per_call_median":143.11916698352434,"ms_per_call_warm_median":143.11916698352434},"own_text/text":{"runs":2,"expected_runs":2,"missing_runs":[],"questions":7,"argmax_equal":7,"questions_non_near_tie":7,"argmax_equal_non_near_tie":7,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.0015869438648223877,"max_abs_dp_non_near_tie":0.0015869438648223877,"mean_of_run_mean_abs_dp":0.000231088128533553,"mean_question_max_abs_dp":0.0003389676234551838,"max_abs_dlogit":0.014174699783325195,"min_pos_cos":0.9777196148126575,"positions":929,"positions_below":{"0.99":2,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0,"runs_with_positions_below_0.99":["own_t01/text","own_t04/text"],"hidden_max_abs_diff":3.3044681549072266,"hidden_max_rel_diff":0.06839162152077685,"worst_run":{"id":"own_t01","arm":"text","max_abs_dp":0.0015869438648223877},"finite_all":true,"tokens":929,"calls":15,"ms_per_call_median":156.22725000139326,"ms_per_call_warm_median":143.55606251046993},"own_text_long/text":{"runs":3,"expected_runs":3,"missing_runs":[],"questions":16,"argmax_equal":16,"questions_non_near_tie":16,"argmax_equal_non_near_tie":16,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.0017313361167907715,"max_abs_dp_non_near_tie":0.0017313361167907715,"mean_of_run_mean_abs_dp":0.00015769999117552362,"mean_question_max_abs_dp":0.00023582599533256143,"max_abs_dlogit":0.0347122848033905,"min_pos_cos":0.9447035349498052,"positions":7486,"positions_below":{"0.99":5,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0,"runs_with_positions_below_0.99":["own_t14/text","own_t15/text","own_t16/text"],"hidden_max_abs_diff":3.4329700469970703,"hidden_max_rel_diff":0.06425275932752302,"worst_run":{"id":"own_t14","arm":"text","max_abs_dp":0.0017313361167907715},"finite_all":true,"tokens":7486,"calls":118,"ms_per_call_median":145.1016874925699,"ms_per_call_warm_median":145.1016874925699},"semif_authored144/text":{"runs":15,"expected_runs":15,"missing_runs":[],"questions":15,"argmax_equal":15,"questions_non_near_tie":15,"argmax_equal_non_near_tie":15,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.007514894008636475,"max_abs_dp_non_near_tie":0.007514894008636475,"mean_of_run_mean_abs_dp":0.0005551291836632621,"mean_question_max_abs_dp":0.0008326997359593709,"max_abs_dlogit":0.05959749221801758,"min_pos_cos":0.9326040782449294,"positions":2863,"positions_below":{"0.99":9,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0,"runs_with_positions_below_0.99":["semif_9473603b44af63beab79/text","semif_fdb9fc214aea2710763a/text","semif_97a35499bd8717f3e786/text","semif_599b3092b6fd18312d23/text","semif_f3d13cf184f916677def/text","semif_0aa1447680e4f25862bb/text","semif_6d64a8459f64eac7f4cc/text","semif_2708193212d8a4a523c7/text"],"hidden_max_abs_diff":7.866986513137817,"hidden_max_rel_diff":0.14911489731397187,"worst_run":{"id":"semif_0aa1447680e4f25862bb","arm":"text","max_abs_dp":0.007514894008636475},"finite_all":true,"tokens":2863,"calls":50,"ms_per_call_median":143.091750505846,"ms_per_call_warm_median":143.091750505846}},"near_ties":[{"id":"img_06","arm":"g448","question_id":"in_red_zone","oracle_top2_margin":0.017518073320388794,"argmax_equal":true,"max_abs_dp":0.0009418129920959473,"probs":[0.5097008347511292,0.490299254655838],"probs_oracle":[0.5087590217590332,0.4912409484386444]},{"id":"img_01","arm":"g256","question_id":"clutter","oracle_top2_margin":0.002184569835662842,"argmax_equal":true,"max_abs_dp":0.00020265579223632812,"probs":[0.49075186252593994,0.48818090558052063,0.021067190915346146],"probs_oracle":[0.4905681312084198,0.48838356137275696,0.02104838564991951]}],"checks":{"all_runs":true,"argmax_non_near_tie":true,"max_abs_dp":true,"mean_of_run_mean_abs_dp":true,"finite":true,"reset_bit_equal_all_processes":true},"result":"PASS","red_arm":null,"timing":{"contended":true,"S":64,"load_seconds":{"first_process":23.777184999984456,"median":23.777184999984456,"all":[23.777184999984456]},"ms_per_call_warm_runs":{"n":389,"median":143.50587499211542,"p10":142.83437498961575,"p90":145.97579141845927},"ms_per_call_all_runs":{"n":396,"median":143.5307704959996,"p10":142.83912499377038,"p90":156.03427099995315},"first_call_of_process_ms":[24096.793792006793],"ms_per_token_warm_runs":{"n":37,"median":2.342158469849882,"p10":2.267422632881654,"p90":2.7486606971461485},"run_decode_seconds_warm_runs":{"n":37,"median":1.1461614159925375,"p10":0.4290339661994949,"p90":2.872779766394524},"run_wall_seconds_warm_runs":{"n":37,"median":1.160374041995965,"p10":0.44032731681363657,"p90":2.9119099416013343},"by_source_arm_warm":{"cc0_photo/g256":{"runs":1,"median_tokens":505.0,"median_calls":8.0,"ms_per_call_median":143.35002050211187,"run_decode_seconds_median":1.1461614159925375},"cc0_photo/g448":{"runs":3,"median_tokens":634.0,"median_calls":10.0,"ms_per_call_median":143.377042026259,"run_decode_seconds_median":1.4341279169893824},"cc0_photo/native":{"runs":3,"median_tokens":1038.0,"median_calls":17.0,"ms_per_call_median":143.7149590055924,"run_decode_seconds_median":2.4426821249944624},"own_image/g256":{"runs":1,"median_tokens":430.0,"median_calls":7.0,"ms_per_call_median":143.45058298204094,"run_decode_seconds_median":1.0033284999954049},"own_image/g448":{"runs":2,"median_tokens":643.5,"median_calls":10.5,"ms_per_call_median":143.17150000715628,"run_decode_seconds_median":1.5031201664969558},"own_image/native":{"runs":5,"median_tokens":696.0,"median_calls":11.0,"ms_per_call_median":143.4589584969217,"run_decode_seconds_median":1.5769217500055674},"own_json/text":{"runs":3,"median_tokens":476.0,"median_calls":8.0,"ms_per_call_median":143.11916698352434,"run_decode_seconds_median":1.1466132909990847},"own_text/text":{"runs":1,"median_tokens":493.0,"median_calls":8.0,"ms_per_call_median":143.55606251046993,"run_decode_seconds_median":1.1618872919934802},"own_text_long/text":{"runs":3,"median_tokens":2494.0,"median_calls":39.0,"ms_per_call_median":145.1016874925699,"run_decode_seconds_median":6.002331166993827},"semif_authored144/text":{"runs":15,"median_tokens":188.0,"median_calls":3.0,"ms_per_call_median":143.091750505846,"run_decode_seconds_median":0.42927045800024644}}},"seconds":{"gpu_processes":109.97547420798219,"scoring":20.751236541982507,"total":130.85557270798017},"runs":[{"id":"own_t01","arm":"text","variant":"base","tokens":436,"calls":7,"padded_tokens":448,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":25.17791704199044,"wall_seconds":25.191991250001593,"finite":true,"index":0,"shard":"shard_00","first_in_process":true,"source":"own_text","hidden":{"max_abs_diff":3.3044681549072266,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.06839162152077685,"min_pos_cos":0.9866572117704246,"min_pos_cos_index":402,"mean_pos_cos":0.9999338970271486,"positions":436,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"department","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":7.802248001098633e-05,"mean_abs_dp":3.901810850948095e-05,"max_abs_dlogit":0.0042008161544799805,"oracle_top2_margin":0.9682950582355261,"near_tie":false,"probs":[0.0020674995612353086,0.00967647135257721,0.009931799955666065,0.9783241152763367]},{"question_id":"wants_refund","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":2.5987625122070312e-05,"mean_abs_dp":2.5975052267313004e-05,"max_abs_dlogit":0.00436854362487793,"oracle_top2_margin":0.9727394469082355,"near_tie":false,"probs":[0.013656244613230228,0.9863437414169312]},{"question_id":"urgency","type":"score","n_options":4,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0015869438648223877,"mean_abs_dp":0.0007934775203466415,"max_abs_dlogit":0.008883237838745117,"oracle_top2_margin":0.07407951354980469,"near_tie":false,"probs":[0.3085280656814575,0.38090166449546814,0.22391775250434875,0.0866524875164032]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0015869438648223877,"mean_abs_dp":0.0003381932619959116,"max_abs_dlogit":0.008883237838745117},{"id":"own_t04","arm":"text","variant":"base","tokens":493,"calls":8,"padded_tokens":512,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":1.1618872919934802,"wall_seconds":1.1743654159945436,"finite":true,"index":1,"shard":"shard_00","first_in_process":false,"source":"own_text","hidden":{"max_abs_diff":2.031962513923645,"ref_absmax":48.97331619262695,"rel_max_abs_diff":0.04149121750161493,"min_pos_cos":0.9777196148126575,"min_pos_cos_index":113,"mean_pos_cos":0.9998830317175706,"positions":493,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"dark_mode","type":"choice","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":7.665157318115234e-05,"mean_abs_dp":5.1110672454039254e-05,"max_abs_dlogit":0.005669772624969482,"oracle_top2_margin":0.9783774577081203,"near_tie":false,"probs":[0.007220098283141851,0.0072571346536278725,0.9855227470397949]},{"question_id":"release_note_owner","type":"choice","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.00018733739852905273,"mean_abs_dp":0.00012492326398690542,"max_abs_dlogit":0.010078221559524536,"oracle_top2_margin":0.9677823921665549,"near_tie":false,"probs":[0.011506867595016956,0.9789782166481018,0.009514969773590565]},{"question_id":"open_items","type":"score","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":3.790855407714844e-05,"mean_abs_dp":2.5309622287750244e-05,"max_abs_dlogit":0.0034308433532714844,"oracle_top2_margin":0.9687048075720668,"near_tie":false,"probs":[0.012095551937818527,0.9807400703430176,0.007164460606873035]},{"question_id":"beta_label_decided","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0003799218684434891,"mean_abs_dp":0.00037989113479852676,"max_abs_dlogit":0.014174699783325195,"oracle_top2_margin":0.9487586636096239,"near_tie":false,"probs":[0.026000559329986572,0.9739994406700134]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0003799218684434891,"mean_abs_dp":0.00012398299507119438,"max_abs_dlogit":0.014174699783325195},{"id":"own_t14","arm":"text","variant":"base","tokens":2389,"calls":38,"padded_tokens":2432,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":6.086956790997647,"wall_seconds":6.098302583995974,"finite":true,"index":2,"shard":"shard_00","first_in_process":false,"source":"own_text_long","hidden":{"max_abs_diff":1.2314882278442383,"ref_absmax":48.31694412231445,"rel_max_abs_diff":0.025487709337054163,"min_pos_cos":0.9876678308827479,"min_pos_cos_index":1532,"mean_pos_cos":0.9999485291593531,"positions":2389,"positions_below":{"0.99":2,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"first_failing_component","type":"choice","n_options":5,"argmax":4,"argmax_oracle":4,"argmax_equal":true,"max_abs_dp":0.00036644935607910156,"mean_abs_dp":0.00014654602855443955,"max_abs_dlogit":0.012581348419189453,"oracle_top2_margin":0.9446960557252169,"near_tie":false,"probs":[0.016855483874678612,0.006241700612008572,0.005250024609267712,0.009449782781302929,0.9622031450271606]},{"question_id":"recovered","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.00017593801021575928,"mean_abs_dp":0.0001759156584739685,"max_abs_dlogit":0.00665283203125,"oracle_top2_margin":0.9060924127697945,"near_tie":false,"probs":[0.9528703093528748,0.04712972790002823]},{"question_id":"severity","type":"score","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":5.283951759338379e-05,"mean_abs_dp":3.9842911064624786e-05,"max_abs_dlogit":0.008376836776733398,"oracle_top2_margin":0.6201646775007248,"near_tie":false,"probs":[0.031833354383707047,0.06412959098815918,0.7620879411697388,0.14194922149181366]},{"question_id":"error_window","type":"choice","n_options":4,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0017313361167907715,"mean_abs_dp":0.0008767074905335903,"max_abs_dlogit":0.014866352081298828,"oracle_top2_margin":0.6222178637981415,"near_tie":false,"probs":[0.16298803687095642,0.7883394956588745,0.04063034802675247,0.008042093366384506]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0017313361167907715,"mean_abs_dp":0.0003167175377408663,"max_abs_dlogit":0.014866352081298828},{"id":"own_t15","arm":"text","variant":"base","tokens":2603,"calls":41,"padded_tokens":2624,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":6.002331166993827,"wall_seconds":6.014046958007384,"finite":true,"index":3,"shard":"shard_00","first_in_process":false,"source":"own_text_long","hidden":{"max_abs_diff":3.4329700469970703,"ref_absmax":53.42914581298828,"rel_max_abs_diff":0.06425275932752302,"min_pos_cos":0.9447035349498052,"min_pos_cos_index":2530,"mean_pos_cos":0.9999227779267673,"positions":2603,"positions_below":{"0.99":2,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"non_renewal_notice","type":"choice","n_options":5,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":8.297152817249298e-06,"mean_abs_dp":4.559103399515152e-06,"max_abs_dlogit":0.005749940872192383,"oracle_top2_margin":0.9844898809678853,"near_tie":false,"probs":[0.004053511191159487,0.002590749878436327,0.002163882600143552,0.9885473251342773,0.002644441556185484]},{"question_id":"auto_renews","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":4.429370164871216e-06,"mean_abs_dp":4.420056939125061e-06,"max_abs_dlogit":0.004403829574584961,"oracle_top2_margin":0.8997148089110851,"near_tie":false,"probs":[0.9498618245124817,0.05013817548751831]},{"question_id":"liability_cap","type":"choice","n_options":4,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":6.270408630371094e-05,"mean_abs_dp":3.132130950689316e-05,"max_abs_dlogit":0.006507933139801025,"oracle_top2_margin":0.9729512641206384,"near_tie":false,"probs":[0.9805378317832947,0.0065671359188854694,0.005231484770774841,0.0076634883880615234]},{"question_id":"assign_without_consent","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":2.792663872241974e-05,"mean_abs_dp":2.788100391626358e-05,"max_abs_dlogit":0.003662109375,"oracle_top2_margin":0.9651009291410446,"near_tie":false,"probs":[0.01747743971645832,0.9825226068496704]},{"question_id":"client_risk","type":"score","n_options":5,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0002937614917755127,"mean_abs_dp":0.00016256198287010193,"max_abs_dlogit":0.0062583088874816895,"oracle_top2_margin":0.11966484785079956,"near_tie":false,"probs":[0.037863291800022125,0.09023147821426392,0.46663418412208557,0.3467881977558136,0.05848286673426628]},{"question_id":"uptime_target","type":"choice","n_options":4,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":6.22868537902832e-05,"mean_abs_dp":3.113748971372843e-05,"max_abs_dlogit":0.009994745254516602,"oracle_top2_margin":0.9738647546619177,"near_tie":false,"probs":[0.007172617129981518,0.9809463620185852,0.006106013432145119,0.005775032099336386]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0002937614917755127,"mean_abs_dp":5.22746704518795e-05,"max_abs_dlogit":0.009994745254516602},{"id":"own_t16","arm":"text","variant":"base","tokens":2494,"calls":39,"padded_tokens":2496,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":5.632826874993043,"wall_seconds":5.645630582992453,"finite":true,"index":4,"shard":"shard_00","first_in_process":false,"source":"own_text_long","hidden":{"max_abs_diff":3.3006343841552734,"ref_absmax":52.57929611206055,"rel_max_abs_diff":0.06277441175934989,"min_pos_cos":0.9774621249063342,"min_pos_cos_index":1714,"mean_pos_cos":0.9999586717873866,"positions":2494,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"water_decision","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":5.439622327685356e-06,"mean_abs_dp":3.6539859138429165e-06,"max_abs_dlogit":0.0023956298828125,"oracle_top2_margin":0.9877108454238623,"near_tie":false,"probs":[0.002710985252633691,0.0028296757955104113,0.0033737129997462034,0.9910855889320374]},{"question_id":"fees_increase","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":4.190206527709961e-05,"mean_abs_dp":4.1880179196596146e-05,"max_abs_dlogit":0.004701018333435059,"oracle_top2_margin":0.9798774998635054,"near_tie":false,"probs":[0.010019377805292606,0.9899806380271912]},{"question_id":"new_chair","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":3.68952751159668e-05,"mean_abs_dp":1.8432270735502243e-05,"max_abs_dlogit":0.0058743953704833984,"oracle_top2_margin":0.9809060669504106,"near_tie":false,"probs":[0.0026575487572699785,0.003915886394679546,0.006235521286725998,0.9871910810470581]},{"question_id":"water_vote_consensus","type":"score","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0007984638214111328,"mean_abs_dp":0.00039923214353621006,"max_abs_dlogit":0.00933748483657837,"oracle_top2_margin":0.7942192256450653,"near_tie":false,"probs":[0.06292495876550674,0.06390862166881561,0.8593569993972778,0.01380942389369011]},{"question_id":"waiting_list_order","type":"choice","n_options":3,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":7.867813110351562e-06,"mean_abs_dp":5.234226894875367e-06,"max_abs_dlogit":0.006229519844055176,"oracle_top2_margin":0.986916062887758,"near_tie":false,"probs":[0.992519736289978,0.0055952416732907295,0.0018850876949727535]},{"question_id":"bonfire_ban","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":9.66787338256836e-05,"mean_abs_dp":9.665545076131821e-05,"max_abs_dlogit":0.0347122848033905,"oracle_top2_margin":0.9465006571263075,"near_tie":false,"probs":[0.973347008228302,0.026653040200471878]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0007984638214111328,"mean_abs_dp":0.00010410776533382504,"max_abs_dlogit":0.0347122848033905},{"id":"own_j08","arm":"text","variant":"base","tokens":473,"calls":8,"padded_tokens":512,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":1.1444105409900658,"wall_seconds":1.1559973749972414,"finite":true,"index":5,"shard":"shard_00","first_in_process":false,"source":"own_json","hidden":{"max_abs_diff":4.445657730102539,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.09201049204743354,"min_pos_cos":0.9584726391881166,"min_pos_cos_index":323,"mean_pos_cos":0.9998796085401992,"positions":473,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"risky","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":5.447864532470703e-05,"mean_abs_dp":5.4463744163513184e-05,"max_abs_dlogit":0.0028564035892486572,"oracle_top2_margin":0.47441771626472473,"near_tie":false,"probs":[0.7372633218765259,0.2627366781234741]},{"question_id":"riskiest_change","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":0.001220613718032837,"mean_abs_dp":0.0006234645843505859,"max_abs_dlogit":0.004977226257324219,"oracle_top2_margin":0.1350460946559906,"near_tie":false,"probs":[0.007399223279207945,0.35096773505210876,0.15365444123744965,0.48797866702079773]},{"question_id":"review","type":"score","n_options":4,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0010154545307159424,"mean_abs_dp":0.000507749617099762,"max_abs_dlogit":0.0070511698722839355,"oracle_top2_margin":0.1977044939994812,"near_tie":false,"probs":[0.08337095379829407,0.4563666582107544,0.25987082719802856,0.2003915160894394]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.001220613718032837,"mean_abs_dp":0.0004633784294128418,"max_abs_dlogit":0.0070511698722839355},{"id":"own_j12","arm":"text","variant":"base","tokens":562,"calls":9,"padded_tokens":576,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":1.286747625010321,"wall_seconds":1.2983184170152526,"finite":true,"index":6,"shard":"shard_00","first_in_process":false,"source":"own_json","hidden":{"max_abs_diff":3.5869650840759277,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.07423837874607986,"min_pos_cos":0.9915981282717938,"min_pos_cos_index":291,"mean_pos_cos":0.9999331793895754,"positions":562,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"coupon_valid","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":8.746795356273651e-05,"mean_abs_dp":8.742418140172958e-05,"max_abs_dlogit":0.007781505584716797,"oracle_top2_margin":0.9615767225623131,"near_tie":false,"probs":[0.0192990954965353,0.9807009696960449]},{"question_id":"total_band","type":"choice","n_options":4,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.0007842183113098145,"mean_abs_dp":0.0003921068273484707,"max_abs_dlogit":0.007659435272216797,"oracle_top2_margin":0.5870811939239502,"near_tie":false,"probs":[0.7358227968215942,0.14766113460063934,0.030691059306263924,0.08582498133182526]},{"question_id":"dessert_items","type":"choice","n_options":4,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.003338932991027832,"mean_abs_dp":0.0016755172982811928,"max_abs_dlogit":0.008608996868133545,"oracle_top2_margin":0.1630445122718811,"near_tie":false,"probs":[0.0482100211083889,0.5379003882408142,0.3682248592376709,0.045664772391319275]},{"question_id":"basket_size","type":"score","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":3.6656856536865234e-05,"mean_abs_dp":2.4436662594477337e-05,"max_abs_dlogit":0.004387855529785156,"oracle_top2_margin":0.6478055715560913,"near_tie":false,"probs":[0.16681331396102905,0.814659833908081,0.018526816740632057]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.003338932991027832,"mean_abs_dp":0.0006552811425465804,"max_abs_dlogit":0.008608996868133545},{"id":"img_03","arm":"native","variant":"base","tokens":501,"calls":8,"padded_tokens":512,"grid":[7,10],"n_image_rows":70,"start":107,"amount":60,"image_tokens":70,"decode_seconds":1.144065792002948,"wall_seconds":1.1624447079957463,"finite":true,"index":7,"shard":"shard_00","first_in_process":false,"source":"own_image","hidden":{"max_abs_diff":6.50660252571106,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.13466527031399245,"min_pos_cos":0.9793232402282466,"min_pos_cos_index":105,"mean_pos_cos":0.9999375712475739,"positions":501,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":1},"questions":[{"question_id":"tens_digit","type":"choice","n_options":10,"argmax":4,"argmax_oracle":4,"argmax_equal":true,"max_abs_dp":0.00012856721878051758,"mean_abs_dp":2.5719963014125823e-05,"max_abs_dlogit":0.005079984664916992,"oracle_top2_margin":0.9546868926845491,"near_tie":false,"probs":[0.004269968718290329,0.004813374951481819,0.004274230450391769,0.0045736259780824184,0.9601517915725708,0.005323696881532669,0.0040421560406684875,0.004428957123309374,0.004045868758112192,0.004076284356415272]},{"question_id":"even","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.00027692317962646484,"mean_abs_dp":0.0002769157290458679,"max_abs_dlogit":0.014441609382629395,"oracle_top2_margin":0.84402646869421,"near_tie":false,"probs":[0.07770984619855881,0.9222901463508606]},{"question_id":"readability","type":"score","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.000591576099395752,"mean_abs_dp":0.0003943890333175659,"max_abs_dlogit":0.0115966796875,"oracle_top2_margin":0.6572192162275314,"near_tie":false,"probs":[0.06456390023231506,0.13863077759742737,0.7968053221702576]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.000591576099395752,"mean_abs_dp":0.00013294654587904612,"max_abs_dlogit":0.014441609382629395},{"id":"img_04","arm":"g448","variant":"base","tokens":690,"calls":11,"padded_tokens":704,"grid":[14,14],"n_image_rows":196,"start":233,"amount":182,"image_tokens":196,"decode_seconds":1.575445624999702,"wall_seconds":1.5962999160110485,"finite":true,"index":8,"shard":"shard_00","first_in_process":false,"source":"own_image","hidden":{"max_abs_diff":3.14107065461576,"ref_absmax":48.31694793701172,"rel_max_abs_diff":0.06500970754010807,"min_pos_cos":0.9838300194602734,"min_pos_cos_index":208,"mean_pos_cos":0.9998685007270461,"positions":690,"positions_below":{"0.99":2,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":2},"questions":[{"question_id":"total","type":"choice","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00012981891632080078,"mean_abs_dp":6.491201929748058e-05,"max_abs_dlogit":0.002858459949493408,"oracle_top2_margin":0.9414124302566051,"near_tie":false,"probs":[0.017000500112771988,0.013121786527335644,0.9582089781761169,0.01166876032948494]},{"question_id":"paid_cash","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.0001232437789440155,"mean_abs_dp":0.00012322328984737396,"max_abs_dlogit":0.009592056274414062,"oracle_top2_margin":0.9361573942005634,"near_tie":false,"probs":[0.967955470085144,0.03204452246427536]},{"question_id":"items","type":"choice","n_options":6,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00010961294174194336,"mean_abs_dp":3.653555177152157e-05,"max_abs_dlogit":0.004731118679046631,"oracle_top2_margin":0.9564370606094599,"near_tie":false,"probs":[0.005974111147224903,0.008289609104394913,0.9653711318969727,0.008802041411399841,0.005905580706894398,0.0056574721820652485]},{"question_id":"legibility","type":"score","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0004953742027282715,"mean_abs_dp":0.0003302345673243205,"max_abs_dlogit":0.0033440589904785156,"oracle_top2_margin":0.7817753255367279,"near_tie":false,"probs":[0.048584945499897,0.08437827229499817,0.8670368194580078]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0004953742027282715,"mean_abs_dp":0.00011440077796578407,"max_abs_dlogit":0.009592056274414062},{"id":"img_04","arm":"native","variant":"base","tokens":1262,"calls":20,"padded_tokens":1280,"grid":[24,32],"n_image_rows":768,"start":805,"amount":736,"image_tokens":768,"decode_seconds":2.87005641599535,"wall_seconds":2.909759958012728,"finite":true,"index":9,"shard":"shard_00","first_in_process":false,"source":"own_image","hidden":{"max_abs_diff":12.449501037597656,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.2576637217161507,"min_pos_cos":0.6492394238482659,"min_pos_cos_index":610,"mean_pos_cos":0.9994959403013532,"positions":1262,"positions_below":{"0.99":6,"0.9":1,"0.5":0},"positions_below_0.99_image_rows":6},"questions":[{"question_id":"total","type":"choice","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":5.27501106262207e-05,"mean_abs_dp":2.926494926214218e-05,"max_abs_dlogit":0.0023975372314453125,"oracle_top2_margin":0.9573134584352374,"near_tie":false,"probs":[0.012957562692463398,0.008554266765713692,0.970180094242096,0.008308090269565582]},{"question_id":"paid_cash","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":5.9626996517181396e-05,"mean_abs_dp":5.961582064628601e-05,"max_abs_dlogit":0.01603877544403076,"oracle_top2_margin":0.9134139567613602,"near_tie":false,"probs":[0.9567666053771973,0.043233416974544525]},{"question_id":"items","type":"choice","n_options":6,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":4.965066909790039e-05,"mean_abs_dp":1.7502810806035995e-05,"max_abs_dlogit":0.0038967132568359375,"oracle_top2_margin":0.9510575840249658,"near_tie":false,"probs":[0.006747474428266287,0.008678160607814789,0.9609953165054321,0.009876844473183155,0.007078386843204498,0.006623858120292425]},{"question_id":"legibility","type":"score","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0005261898040771484,"mean_abs_dp":0.0003507894774278005,"max_abs_dlogit":0.0034338831901550293,"oracle_top2_margin":0.819927416741848,"near_tie":false,"probs":[0.04532945156097412,0.06695050746202469,0.8877200484275818]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0005261898040771484,"mean_abs_dp":9.291178236405054e-05,"max_abs_dlogit":0.01603877544403076},{"id":"img_06","arm":"g448","variant":"base","tokens":597,"calls":10,"padded_tokens":640,"grid":[14,14],"n_image_rows":196,"start":233,"amount":182,"image_tokens":196,"decode_seconds":1.4307947079942096,"wall_seconds":1.4508748340012971,"finite":true,"index":10,"shard":"shard_00","first_in_process":false,"source":"own_image","hidden":{"max_abs_diff":2.5992231369018555,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.05379536938888395,"min_pos_cos":0.9766007468198847,"min_pos_cos_index":178,"mean_pos_cos":0.9998973954226124,"positions":597,"positions_below":{"0.99":2,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":2},"questions":[{"question_id":"in_red_zone","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.0009418129920959473,"mean_abs_dp":0.0009417533874511719,"max_abs_dlogit":0.011691451072692871,"oracle_top2_margin":0.017518073320388794,"near_tie":true,"probs":[0.5097008347511292,0.490299254655838]},{"question_id":"reading","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":0.0018284618854522705,"mean_abs_dp":0.0009142239578068256,"max_abs_dlogit":0.013373851776123047,"oracle_top2_margin":0.05165323615074158,"near_tie":false,"probs":[0.015895184129476547,0.039822667837142944,0.44458696246147156,0.49969518184661865]},{"question_id":"alarm","type":"score","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0007908940315246582,"mean_abs_dp":0.0004452615976333618,"max_abs_dlogit":0.02442002296447754,"oracle_top2_margin":0.3209385871887207,"near_tie":false,"probs":[0.0994485467672348,0.24849757552146912,0.568744957447052,0.08330885320901871]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0018284618854522705,"mean_abs_dp":0.0007321448996663093,"max_abs_dlogit":0.02442002296447754},{"id":"img_07","arm":"native","variant":"base","tokens":662,"calls":11,"padded_tokens":704,"grid":[15,20],"n_image_rows":300,"start":337,"amount":280,"image_tokens":300,"decode_seconds":1.57571029101382,"wall_seconds":1.598851165996166,"finite":true,"index":11,"shard":"shard_00","first_in_process":false,"source":"own_image","hidden":{"max_abs_diff":5.537443161010742,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.11460685929088438,"min_pos_cos":0.9848827502966294,"min_pos_cos_index":85,"mean_pos_cos":0.9998179558883522,"positions":662,"positions_below":{"0.99":2,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":2},"questions":[{"question_id":"odd_direction","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":4.41279262304306e-05,"mean_abs_dp":2.2062798961997032e-05,"max_abs_dlogit":0.006595611572265625,"oracle_top2_margin":0.9635526714846492,"near_tie":false,"probs":[0.007775760721415281,0.00985729694366455,0.008996577933430672,0.973370373249054]},{"question_id":"odd_position","type":"choice","n_options":4,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.00026392191648483276,"mean_abs_dp":0.00013193488121032715,"max_abs_dlogit":0.008801937103271484,"oracle_top2_margin":0.8184187300503254,"near_tie":false,"probs":[0.045024026185274124,0.866546094417572,0.04846483841538429,0.03996502608060837]},{"question_id":"all_same","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":1.8656253814697266e-05,"mean_abs_dp":1.8640421330928802e-05,"max_abs_dlogit":0.004556417465209961,"oracle_top2_margin":0.9710730528458953,"near_tie":false,"probs":[0.014444838277995586,0.9855551719665527]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00026392191648483276,"mean_abs_dp":6.532715633511543e-05,"max_abs_dlogit":0.008801937103271484},{"id":"img_08","arm":"native","variant":"base","tokens":1226,"calls":20,"padded_tokens":1280,"grid":[24,32],"n_image_rows":768,"start":805,"amount":736,"image_tokens":768,"decode_seconds":2.8768647919932846,"wall_seconds":2.9151349169842433,"finite":true,"index":12,"shard":"shard_00","first_in_process":false,"source":"own_image","hidden":{"max_abs_diff":3.0692062377929688,"ref_absmax":50.38888931274414,"rel_max_abs_diff":0.060910376863907545,"min_pos_cos":0.9782105673340956,"min_pos_cos_index":762,"mean_pos_cos":0.9997810354354155,"positions":1226,"positions_below":{"0.99":3,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":3},"questions":[{"question_id":"largest_room","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":8.685141801834106e-05,"mean_abs_dp":4.340102896094322e-05,"max_abs_dlogit":0.010195732116699219,"oracle_top2_margin":0.8977233991026878,"near_tie":false,"probs":[0.011415437795221806,0.03461568057537079,0.021725855767726898,0.9322431087493896]},{"question_id":"rooms","type":"choice","n_options":5,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0014417171478271484,"mean_abs_dp":0.00060297641903162,"max_abs_dlogit":0.01251530647277832,"oracle_top2_margin":0.14287957549095154,"near_tie":false,"probs":[0.010423804633319378,0.040521010756492615,0.5202860832214355,0.3800365924835205,0.048732466995716095]},{"question_id":"bath_next_to_bed","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":3.695487976074219e-05,"mean_abs_dp":3.692507743835449e-05,"max_abs_dlogit":0.0058152079582214355,"oracle_top2_margin":0.384887158870697,"near_tie":false,"probs":[0.30759334564208984,0.6924066543579102]},{"question_id":"complexity","type":"score","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0005561411380767822,"mean_abs_dp":0.00037078559398651123,"max_abs_dlogit":0.006466537714004517,"oracle_top2_margin":0.4244216978549957,"near_tie":false,"probs":[0.26027363538742065,0.6837853789329529,0.05594092980027199]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0014417171478271484,"mean_abs_dp":0.00031247808198843686,"max_abs_dlogit":0.01251530647277832},{"id":"img_09","arm":"native","variant":"base","tokens":696,"calls":11,"padded_tokens":704,"grid":[15,20],"n_image_rows":300,"start":337,"amount":280,"image_tokens":300,"decode_seconds":1.5769217500055674,"wall_seconds":1.6005035839916673,"finite":true,"index":13,"shard":"shard_00","first_in_process":false,"source":"own_image","hidden":{"max_abs_diff":3.865550994873047,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.08000419075548394,"min_pos_cos":0.9165806653848869,"min_pos_cos_index":184,"mean_pos_cos":0.9998092620388436,"positions":696,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":1},"questions":[{"question_id":"largest","type":"choice","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":1.929979771375656e-05,"mean_abs_dp":1.285504549741745e-05,"max_abs_dlogit":0.002392292022705078,"oracle_top2_margin":0.96602054964751,"near_tie":false,"probs":[0.01169352512806654,0.010588360950350761,0.9777180552482605]},{"question_id":"smallest","type":"choice","n_options":3,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.00014674663543701172,"mean_abs_dp":9.784164528052013e-05,"max_abs_dlogit":0.004866123199462891,"oracle_top2_margin":0.9375971518456936,"near_tie":false,"probs":[0.9601291418075562,0.022699980065226555,0.017170879989862442]},{"question_id":"same_size","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":2.9921531677246094e-05,"mean_abs_dp":2.9918737709522247e-05,"max_abs_dlogit":0.0022270679473876953,"oracle_top2_margin":0.9730345876887441,"near_tie":false,"probs":[0.013452806510031223,0.9865472316741943]},{"question_id":"spread","type":"score","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":7.817894220352173e-05,"mean_abs_dp":5.211681127548218e-05,"max_abs_dlogit":0.000761866569519043,"oracle_top2_margin":0.3400270342826843,"near_tie":false,"probs":[0.08156803995370865,0.28917667269706726,0.6292552351951599]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00014674663543701172,"mean_abs_dp":4.9843452870845795e-05,"max_abs_dlogit":0.004866123199462891},{"id":"photo_01","arm":"g448","variant":"base","tokens":688,"calls":11,"padded_tokens":704,"grid":[14,14],"n_image_rows":196,"start":233,"amount":182,"image_tokens":196,"decode_seconds":1.5769079169840552,"wall_seconds":1.596286957996199,"finite":true,"index":14,"shard":"shard_00","first_in_process":false,"source":"cc0_photo","hidden":{"max_abs_diff":2.1825122833251953,"ref_absmax":48.66167068481445,"rel_max_abs_diff":0.044850747058429265,"min_pos_cos":0.986123553011467,"min_pos_cos_index":162,"mean_pos_cos":0.999918485627392,"positions":688,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":1},"questions":[{"question_id":"limes","type":"choice","n_options":6,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":0.004601955413818359,"mean_abs_dp":0.0015477196623881657,"max_abs_dlogit":0.015121936798095703,"oracle_top2_margin":0.056382983922958374,"near_tie":false,"probs":[0.013759546913206577,0.0406072773039341,0.4085417687892914,0.45635291934013367,0.056228723376989365,0.024509841576218605]},{"question_id":"has_banana","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":1.1920928955078125e-06,"mean_abs_dp":1.1860392987728119e-06,"max_abs_dlogit":0.0008599758148193359,"oracle_top2_margin":0.992401025025174,"near_tie":false,"probs":[0.003798296907916665,0.9962016940116882]},{"question_id":"kinds","type":"choice","n_options":5,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0018820762634277344,"mean_abs_dp":0.0007589731365442276,"max_abs_dlogit":0.010043978691101074,"oracle_top2_margin":0.23297813534736633,"near_tie":false,"probs":[0.023915551602840424,0.13909004628658295,0.5054147243499756,0.27430328726768494,0.057276397943496704]},{"question_id":"freshness","type":"score","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00011020898818969727,"mean_abs_dp":7.346707085768382e-05,"max_abs_dlogit":0.0035753250122070312,"oracle_top2_margin":0.6520294547080994,"near_tie":false,"probs":[0.009119652211666107,0.16933786869049072,0.821542501449585]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.004601955413818359,"mean_abs_dp":0.0008314973092637956,"max_abs_dlogit":0.015121936798095703},{"id":"photo_02","arm":"g448","variant":"base","tokens":631,"calls":10,"padded_tokens":640,"grid":[14,14],"n_image_rows":196,"start":233,"amount":182,"image_tokens":196,"decode_seconds":1.4337690410029609,"wall_seconds":1.4542132090136874,"finite":true,"index":15,"shard":"shard_00","first_in_process":false,"source":"cc0_photo","hidden":{"max_abs_diff":20.76478672027588,"ref_absmax":49.467185974121094,"rel_max_abs_diff":0.4197689096594061,"min_pos_cos":0.6234456456156695,"min_pos_cos_index":163,"mean_pos_cos":0.9993589545675639,"positions":631,"positions_below":{"0.99":1,"0.9":1,"0.5":0},"positions_below_0.99_image_rows":1},"questions":[{"question_id":"animal","type":"choice","n_options":4,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":8.406490087509155e-05,"mean_abs_dp":4.20217402279377e-05,"max_abs_dlogit":0.003413677215576172,"oracle_top2_margin":0.9437561091035604,"near_tie":false,"probs":[0.9616857767105103,0.010354532860219479,0.017776580527424812,0.01018309872597456]},{"question_id":"eye_color","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":6.455183029174805e-05,"mean_abs_dp":3.229163121432066e-05,"max_abs_dlogit":0.0013995170593261719,"oracle_top2_margin":0.9384196568280458,"near_tie":false,"probs":[0.007051872555166483,0.012083815410733223,0.021169070154428482,0.9596951603889465]},{"question_id":"background","type":"score","n_options":3,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.0006910562515258789,"mean_abs_dp":0.0004607116182645162,"max_abs_dlogit":0.004479646682739258,"oracle_top2_margin":0.5623989552259445,"near_tie":false,"probs":[0.7500767111778259,0.18637102842330933,0.06355227530002594]},{"question_id":"striped","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":2.1576881408691406e-05,"mean_abs_dp":2.1569430828094482e-05,"max_abs_dlogit":0.0023080110549926758,"oracle_top2_margin":0.8637556880712509,"near_tie":false,"probs":[0.9318994283676147,0.06810060143470764]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0006910562515258789,"mean_abs_dp":0.000132502092478367,"max_abs_dlogit":0.004479646682739258},{"id":"photo_02","arm":"native","variant":"base","tokens":1035,"calls":17,"padded_tokens":1088,"grid":[20,30],"n_image_rows":600,"start":637,"amount":570,"image_tokens":600,"decode_seconds":2.442915833002189,"wall_seconds":2.475707290985156,"finite":true,"index":16,"shard":"shard_00","first_in_process":false,"source":"cc0_photo","hidden":{"max_abs_diff":4.344765663146973,"ref_absmax":48.347782135009766,"rel_max_abs_diff":0.08986483911535677,"min_pos_cos":0.9793512448671396,"min_pos_cos_index":195,"mean_pos_cos":0.9999094472216786,"positions":1035,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":1},"questions":[{"question_id":"animal","type":"choice","n_options":4,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.00014662742614746094,"mean_abs_dp":7.330975495278835e-05,"max_abs_dlogit":0.004131913185119629,"oracle_top2_margin":0.949346523731947,"near_tie":false,"probs":[0.9658992290496826,0.009067459963262081,0.016318004578351974,0.008715310133993626]},{"question_id":"eye_color","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":9.971857070922852e-05,"mean_abs_dp":4.985777195543051e-05,"max_abs_dlogit":0.00211411714553833,"oracle_top2_margin":0.9311072155833244,"near_tie":false,"probs":[0.006742448545992374,0.013341756537556648,0.02431364357471466,0.9556021094322205]},{"question_id":"background","type":"score","n_options":3,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.00019758939743041992,"mean_abs_dp":0.00013173123200734457,"max_abs_dlogit":0.004959583282470703,"oracle_top2_margin":0.6139450818300247,"near_tie":false,"probs":[0.7790936231613159,0.16547581553459167,0.05543055757880211]},{"question_id":"striped","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":1.0967254638671875e-05,"mean_abs_dp":1.093745231628418e-05,"max_abs_dlogit":0.004790782928466797,"oracle_top2_margin":0.8699666932225227,"near_tie":false,"probs":[0.9349724054336548,0.06502758711576462]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00019758939743041992,"mean_abs_dp":6.997990063749827e-05,"max_abs_dlogit":0.004959583282470703},{"id":"photo_03","arm":"g256","variant":"base","tokens":505,"calls":8,"padded_tokens":512,"grid":[8,8],"n_image_rows":64,"start":101,"amount":56,"image_tokens":64,"decode_seconds":1.1461614159925375,"wall_seconds":1.160374041995965,"finite":true,"index":17,"shard":"shard_00","first_in_process":false,"source":"cc0_photo","hidden":{"max_abs_diff":1.9747991561889648,"ref_absmax":48.31694793701172,"rel_max_abs_diff":0.040871769441302615,"min_pos_cos":0.9700445583965338,"min_pos_cos_index":116,"mean_pos_cos":0.9999261099745526,"positions":505,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"snow_visible","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":5.644559860229492e-05,"mean_abs_dp":5.644187331199646e-05,"max_abs_dlogit":0.001796722412109375,"oracle_top2_margin":0.8396650552749634,"near_tie":false,"probs":[0.9197760820388794,0.08022391051054001]},{"question_id":"reflection","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":1.893937587738037e-05,"mean_abs_dp":1.89170241355896e-05,"max_abs_dlogit":0.002003192901611328,"oracle_top2_margin":0.903554379940033,"near_tie":false,"probs":[0.9517582654953003,0.04824171960353851]},{"question_id":"sky","type":"choice","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":5.698204040527344e-05,"mean_abs_dp":2.8484035283327103e-05,"max_abs_dlogit":0.00260162353515625,"oracle_top2_margin":0.905474217608571,"near_tie":false,"probs":[0.021484769880771637,0.02614712528884411,0.9317183494567871,0.02064979448914528]},{"question_id":"wildness","type":"score","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":3.8743019104003906e-05,"mean_abs_dp":1.9359635189175606e-05,"max_abs_dlogit":0.004570960998535156,"oracle_top2_margin":0.8537349253892899,"near_tie":false,"probs":[0.012335562147200108,0.01298100221902132,0.06050663813948631,0.9141768217086792]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":5.698204040527344e-05,"mean_abs_dp":2.8507706398765247e-05,"max_abs_dlogit":0.004570960998535156},{"id":"photo_03","arm":"native","variant":"base","tokens":1041,"calls":17,"padded_tokens":1088,"grid":[20,30],"n_image_rows":600,"start":637,"amount":570,"image_tokens":600,"decode_seconds":2.441700374998618,"wall_seconds":2.4750891670119017,"finite":true,"index":18,"shard":"shard_00","first_in_process":false,"source":"cc0_photo","hidden":{"max_abs_diff":4.143296957015991,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.08575261859056398,"min_pos_cos":0.9746840734197253,"min_pos_cos_index":617,"mean_pos_cos":0.999896374881662,"positions":1041,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":1},"questions":[{"question_id":"snow_visible","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.00016373395919799805,"mean_abs_dp":0.00016373395919799805,"max_abs_dlogit":0.008470535278320312,"oracle_top2_margin":0.8728517517447472,"near_tie":false,"probs":[0.9365895986557007,0.06341037899255753]},{"question_id":"reflection","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.00015737861394882202,"mean_abs_dp":0.00015733763575553894,"max_abs_dlogit":0.011842012405395508,"oracle_top2_margin":0.911050520837307,"near_tie":false,"probs":[0.9556825757026672,0.04431737959384918]},{"question_id":"sky","type":"choice","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":4.07099723815918e-05,"mean_abs_dp":2.0363833755254745e-05,"max_abs_dlogit":0.001806497573852539,"oracle_top2_margin":0.9153185542672873,"near_tie":false,"probs":[0.02020496129989624,0.022752344608306885,0.9381298422813416,0.0189127828925848]},{"question_id":"wildness","type":"score","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":0.00020879507064819336,"mean_abs_dp":0.0001044136006385088,"max_abs_dlogit":0.005450904369354248,"oracle_top2_margin":0.8406233042478561,"near_tie":false,"probs":[0.013047873973846436,0.014019935391843319,0.06598377972841263,0.9069483876228333]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00020879507064819336,"mean_abs_dp":9.510441062351067e-05,"max_abs_dlogit":0.011842012405395508},{"id":"photo_04","arm":"g448","variant":"base","tokens":634,"calls":10,"padded_tokens":640,"grid":[14,14],"n_image_rows":196,"start":233,"amount":182,"image_tokens":196,"decode_seconds":1.4341279169893824,"wall_seconds":1.4540565420174971,"finite":true,"index":19,"shard":"shard_00","first_in_process":false,"source":"cc0_photo","hidden":{"max_abs_diff":20.050286293029785,"ref_absmax":48.31694412231445,"rel_max_abs_diff":0.41497422192662764,"min_pos_cos":0.37844254882577144,"min_pos_cos_index":134,"mean_pos_cos":0.9986890973962101,"positions":634,"positions_below":{"0.99":4,"0.9":1,"0.5":1},"positions_below_0.99_image_rows":4},"questions":[{"question_id":"drink","type":"choice","n_options":4,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.00013649463653564453,"mean_abs_dp":6.825383752584457e-05,"max_abs_dlogit":0.0056188106536865234,"oracle_top2_margin":0.9513880060985684,"near_tie":false,"probs":[0.965700626373291,0.01003585197031498,0.010135513730347157,0.014127993024885654]},{"question_id":"milk_added","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":2.2649765014648438e-06,"mean_abs_dp":2.2391323000192642e-06,"max_abs_dlogit":0.0023543834686279297,"oracle_top2_margin":0.989408596418798,"near_tie":false,"probs":[0.005297922994941473,0.9947020411491394]},{"question_id":"cup_color","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":1.9490718841552734e-05,"mean_abs_dp":1.5445752069354057e-05,"max_abs_dlogit":0.004315376281738281,"oracle_top2_margin":0.970035164617002,"near_tie":false,"probs":[0.007261557504534721,0.007222969550639391,0.007723244372755289,0.9777921438217163]},{"question_id":"beans","type":"score","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":4.8481859266757965e-05,"mean_abs_dp":3.230726967255274e-05,"max_abs_dlogit":0.004063844680786133,"oracle_top2_margin":0.9172959104180336,"near_tie":false,"probs":[0.01127413846552372,0.035693708807229996,0.9530321359634399]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00013649463653564453,"mean_abs_dp":3.355372553834548e-05,"max_abs_dlogit":0.0056188106536865234},{"id":"photo_04","arm":"native","variant":"base","tokens":1038,"calls":17,"padded_tokens":1088,"grid":[20,30],"n_image_rows":600,"start":637,"amount":570,"image_tokens":600,"decode_seconds":2.4426821249944624,"wall_seconds":2.4755862499878276,"finite":true,"index":20,"shard":"shard_00","first_in_process":false,"source":"cc0_photo","hidden":{"max_abs_diff":16.05294418334961,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.33224314213814116,"min_pos_cos":0.6753120540250005,"min_pos_cos_index":468,"mean_pos_cos":0.9990784275112199,"positions":1038,"positions_below":{"0.99":10,"0.9":2,"0.5":0},"positions_below_0.99_image_rows":10},"questions":[{"question_id":"drink","type":"choice","n_options":4,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.00010836124420166016,"mean_abs_dp":5.417666397988796e-05,"max_abs_dlogit":0.0034767985343933105,"oracle_top2_margin":0.951157646253705,"near_tie":false,"probs":[0.9651314616203308,0.010546413250267506,0.010492195375263691,0.013829939067363739]},{"question_id":"milk_added","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":4.932750016450882e-06,"mean_abs_dp":4.880363121628761e-06,"max_abs_dlogit":0.0017285346984863281,"oracle_top2_margin":0.9896592232398689,"near_tie":false,"probs":[0.005175304599106312,0.9948247671127319]},{"question_id":"cup_color","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":3.808736801147461e-05,"mean_abs_dp":1.9046710804104805e-05,"max_abs_dlogit":0.0013712048530578613,"oracle_top2_margin":0.970263154245913,"near_tie":false,"probs":[0.00686913076788187,0.007277721539139748,0.007768542040139437,0.9780845642089844]},{"question_id":"beans","type":"score","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":6.663799285888672e-05,"mean_abs_dp":4.4445196787516274e-05,"max_abs_dlogit":0.0029382556676864624,"oracle_top2_margin":0.9172757267951965,"near_tie":false,"probs":[0.012123537249863148,0.03525935858488083,0.9526171088218689]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00010836124420166016,"mean_abs_dp":3.353767813398288e-05,"max_abs_dlogit":0.0034767985343933105},{"id":"semif_9473603b44af63beab79","arm":"text","variant":"base","tokens":190,"calls":3,"padded_tokens":192,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.4289835410017986,"wall_seconds":0.440368750016205,"finite":true,"index":21,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":4.679834842681885,"ref_absmax":48.31694793701172,"rel_max_abs_diff":0.09685700447765742,"min_pos_cos":0.9791982303024819,"min_pos_cos_index":173,"mean_pos_cos":0.999857416305208,"positions":190,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.00020587444305419922,"mean_abs_dp":0.00013726639250914255,"max_abs_dlogit":0.009129643440246582,"oracle_top2_margin":0.9331219494342804,"near_tie":false,"probs":[0.023544324561953545,0.9569852352142334,0.019470389932394028]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00020587444305419922,"mean_abs_dp":0.00013726639250914255,"max_abs_dlogit":0.009129643440246582},{"id":"semif_fdb9fc214aea2710763a","arm":"text","variant":"base","tokens":168,"calls":3,"padded_tokens":192,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.4297267079818994,"wall_seconds":0.4405723749950994,"finite":true,"index":22,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":4.689628601074219,"ref_absmax":48.31687927246094,"rel_max_abs_diff":0.09705984061241214,"min_pos_cos":0.9710214073324903,"min_pos_cos_index":151,"mean_pos_cos":0.9997656038932264,"positions":168,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.0011172890663146973,"mean_abs_dp":0.000744828333457311,"max_abs_dlogit":0.01824665069580078,"oracle_top2_margin":0.8761075809597969,"near_tie":false,"probs":[0.9164733290672302,0.04209437593817711,0.04143219813704491]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0011172890663146973,"mean_abs_dp":0.000744828333457311,"max_abs_dlogit":0.01824665069580078},{"id":"semif_97a35499bd8717f3e786","arm":"text","variant":"base","tokens":203,"calls":4,"padded_tokens":256,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.5734874999907333,"wall_seconds":0.5850136670051143,"finite":true,"index":23,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":4.465639114379883,"ref_absmax":48.316925048828125,"rel_max_abs_diff":0.09242390963139721,"min_pos_cos":0.9632720406266345,"min_pos_cos_index":63,"mean_pos_cos":0.9997453526727355,"positions":203,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0001971125602722168,"mean_abs_dp":0.00013139843940734863,"max_abs_dlogit":0.005466938018798828,"oracle_top2_margin":0.9219651073217392,"near_tie":false,"probs":[0.02705015055835247,0.9486802220344543,0.024269621819257736]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0001971125602722168,"mean_abs_dp":0.00013139843940734863,"max_abs_dlogit":0.005466938018798828},{"id":"semif_599b3092b6fd18312d23","arm":"text","variant":"base","tokens":187,"calls":3,"padded_tokens":192,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.4290675829979591,"wall_seconds":0.4405202920024749,"finite":true,"index":24,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":1.5794072151184082,"ref_absmax":48.31692123413086,"rel_max_abs_diff":0.03268849038342124,"min_pos_cos":0.9670354040238728,"min_pos_cos_index":108,"mean_pos_cos":0.999765211807583,"positions":187,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.00035190582275390625,"mean_abs_dp":0.00023460264007250467,"max_abs_dlogit":0.005724430084228516,"oracle_top2_margin":0.9458253737539053,"near_tie":false,"probs":[0.9636124968528748,0.018070418387651443,0.018317118287086487]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00035190582275390625,"mean_abs_dp":0.00023460264007250467,"max_abs_dlogit":0.005724430084228516},{"id":"semif_f3d13cf184f916677def","arm":"text","variant":"base","tokens":208,"calls":4,"padded_tokens":256,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.5713978749990929,"wall_seconds":0.5823401250236202,"finite":true,"index":25,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":7.204771041870117,"ref_absmax":48.31690979003906,"rel_max_abs_diff":0.14911489731397187,"min_pos_cos":0.9719749167161514,"min_pos_cos_index":148,"mean_pos_cos":0.999820612418507,"positions":208,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":8.83936882019043e-05,"mean_abs_dp":5.893409252166748e-05,"max_abs_dlogit":0.0022640228271484375,"oracle_top2_margin":0.8051065430045128,"near_tie":false,"probs":[0.874390721321106,0.06941606104373932,0.05619320645928383]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":8.83936882019043e-05,"mean_abs_dp":5.893409252166748e-05,"max_abs_dlogit":0.0022640228271484375},{"id":"semif_0aa1447680e4f25862bb","arm":"text","variant":"base","tokens":208,"calls":4,"padded_tokens":256,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.572206750017358,"wall_seconds":0.5817706669913605,"finite":true,"index":26,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":7.866986513137817,"ref_absmax":54.96559143066406,"rel_max_abs_diff":0.1431256593147291,"min_pos_cos":0.9496045187449393,"min_pos_cos_index":189,"mean_pos_cos":0.9995464544287133,"positions":208,"positions_below":{"0.99":2,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.007514894008636475,"mean_abs_dp":0.005009929339090983,"max_abs_dlogit":0.05959749221801758,"oracle_top2_margin":0.695934608578682,"near_tie":false,"probs":[0.09476887434720993,0.09779907763004303,0.8074320554733276]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.007514894008636475,"mean_abs_dp":0.005009929339090983,"max_abs_dlogit":0.05959749221801758},{"id":"semif_6d64a8459f64eac7f4cc","arm":"text","variant":"base","tokens":207,"calls":4,"padded_tokens":256,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.572595082980115,"wall_seconds":0.5845217079913709,"finite":true,"index":27,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":5.5497589111328125,"ref_absmax":50.43628692626953,"rel_max_abs_diff":0.11003504122430241,"min_pos_cos":0.9326040782449294,"min_pos_cos_index":83,"mean_pos_cos":0.9996391424853833,"positions":207,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":5.048513412475586e-05,"mean_abs_dp":3.363254169623057e-05,"max_abs_dlogit":0.0020279884338378906,"oracle_top2_margin":0.9439202304929495,"near_tie":false,"probs":[0.018820978701114655,0.9626902937889099,0.018488667905330658]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":5.048513412475586e-05,"mean_abs_dp":3.363254169623057e-05,"max_abs_dlogit":0.0020279884338378906},{"id":"semif_2708193212d8a4a523c7","arm":"text","variant":"base","tokens":207,"calls":4,"padded_tokens":256,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.5727762920141686,"wall_seconds":0.5837709170009475,"finite":true,"index":28,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":4.081653118133545,"ref_absmax":48.316890716552734,"rel_max_abs_diff":0.0844767338626619,"min_pos_cos":0.98445278471852,"min_pos_cos_index":117,"mean_pos_cos":0.9998618988905007,"positions":207,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.000438690185546875,"mean_abs_dp":0.00029244522253672284,"max_abs_dlogit":0.005296230316162109,"oracle_top2_margin":0.5487232059240341,"near_tie":false,"probs":[0.1314423680305481,0.7089253067970276,0.1596323698759079]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.000438690185546875,"mean_abs_dp":0.00029244522253672284,"max_abs_dlogit":0.005296230316162109},{"id":"own_j01","arm":"text","variant":"base","tokens":476,"calls":8,"padded_tokens":512,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":1.1466132909990847,"wall_seconds":1.158234167000046,"finite":true,"index":29,"shard":"shard_00","first_in_process":false,"source":"own_json","hidden":{"max_abs_diff":1.4357128143310547,"ref_absmax":49.30904769897461,"rel_max_abs_diff":0.029116620201142328,"min_pos_cos":0.999094952914686,"min_pos_cos_index":179,"mean_pos_cos":0.9999746307780836,"positions":476,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"status","type":"choice","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0005807802081108093,"mean_abs_dp":0.00029038265347480774,"max_abs_dlogit":0.016298532485961914,"oracle_top2_margin":0.83003219217062,"near_tie":false,"probs":[0.012074192985892296,0.07337257266044617,0.902336061000824,0.012217206880450249]},{"question_id":"large","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":3.366731107234955e-06,"mean_abs_dp":3.352295607328415e-06,"max_abs_dlogit":0.005334258079528809,"oracle_top2_margin":0.9913223842158914,"near_tie":false,"probs":[0.004335448145866394,0.9956645369529724]},{"question_id":"priority","type":"score","n_options":4,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0009830892086029053,"mean_abs_dp":0.000522444024682045,"max_abs_dlogit":0.015638113021850586,"oracle_top2_margin":0.19878698885440826,"near_tie":false,"probs":[0.24802164733409882,0.4479719400405884,0.23643551766872406,0.06757092475891113]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0009830892086029053,"mean_abs_dp":0.00032580113038420676,"max_abs_dlogit":0.016298532485961914},{"id":"img_01","arm":"g256","variant":"base","tokens":430,"calls":7,"padded_tokens":448,"grid":[8,8],"n_image_rows":64,"start":101,"amount":56,"image_tokens":64,"decode_seconds":1.0033284999954049,"wall_seconds":1.0188094160112087,"finite":true,"index":30,"shard":"shard_00","first_in_process":false,"source":"own_image","hidden":{"max_abs_diff":0.6985244750976562,"ref_absmax":48.31694793701172,"rel_max_abs_diff":0.014457131605421066,"min_pos_cos":0.999530785693714,"min_pos_cos_index":297,"mean_pos_cos":0.9999846967322968,"positions":430,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"circles","type":"choice","n_options":6,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00043833255767822266,"mean_abs_dp":0.00014609765882293382,"max_abs_dlogit":0.01240241527557373,"oracle_top2_margin":0.9245060188695788,"near_tie":false,"probs":[0.009116161614656448,0.01352724153548479,0.9389314651489258,0.011302697472274303,0.01386208925396204,0.01326040830463171]},{"question_id":"more_circles","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.0015357136726379395,"mean_abs_dp":0.0015357062220573425,"max_abs_dlogit":0.008781194686889648,"oracle_top2_margin":0.6116609573364258,"near_tie":false,"probs":[0.8073661923408508,0.19263382256031036]},{"question_id":"clutter","type":"score","n_options":3,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.00020265579223632812,"mean_abs_dp":0.00013506412506103516,"max_abs_dlogit":0.004168987274169922,"oracle_top2_margin":0.002184569835662842,"near_tie":true,"probs":[0.49075186252593994,0.48818090558052063,0.021067190915346146]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0015357136726379395,"mean_abs_dp":0.00039574461565776306,"max_abs_dlogit":0.01240241527557373},{"id":"semif_a3f18f3a63d45345942b","arm":"text","variant":"base","tokens":185,"calls":3,"padded_tokens":192,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.42918266699416563,"wall_seconds":0.4403965419915039,"finite":true,"index":31,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":1.2211074829101562,"ref_absmax":48.31694793701172,"rel_max_abs_diff":0.02527286045679148,"min_pos_cos":0.9986475274404417,"min_pos_cos_index":137,"mean_pos_cos":0.9999598629786692,"positions":185,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":9.66787338256836e-05,"mean_abs_dp":6.445621450742085e-05,"max_abs_dlogit":0.001970529556274414,"oracle_top2_margin":0.9059995450079441,"near_tie":false,"probs":[0.9378738403320312,0.030100984498858452,0.032025132328271866]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":9.66787338256836e-05,"mean_abs_dp":6.445621450742085e-05,"max_abs_dlogit":0.001970529556274414},{"id":"semif_f40beba9088c8db8bbd6","arm":"text","variant":"base","tokens":183,"calls":3,"padded_tokens":192,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.4286149999825284,"wall_seconds":0.43998533400008455,"finite":true,"index":32,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":0.6315784454345703,"ref_absmax":48.316871643066406,"rel_max_abs_diff":0.013071592260779231,"min_pos_cos":0.9991752839927844,"min_pos_cos_index":166,"mean_pos_cos":0.9999630272214562,"positions":183,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00015801191329956055,"mean_abs_dp":0.00010534748435020447,"max_abs_dlogit":0.0034728050231933594,"oracle_top2_margin":0.8385190591216087,"near_tie":false,"probs":[0.03431006893515587,0.06368239223957062,0.9020075798034668]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00015801191329956055,"mean_abs_dp":0.00010534748435020447,"max_abs_dlogit":0.0034728050231933594},{"id":"semif_0b43ea8e24e74d621f1b","arm":"text","variant":"base","tokens":177,"calls":3,"padded_tokens":192,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.42927045800024644,"wall_seconds":0.441518375009764,"finite":true,"index":33,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":0.833582878112793,"ref_absmax":48.31693649291992,"rel_max_abs_diff":0.01725239509410828,"min_pos_cos":0.9996133414734085,"min_pos_cos_index":115,"mean_pos_cos":0.9999783340020545,"positions":177,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00016760826110839844,"mean_abs_dp":0.00011171400547027588,"max_abs_dlogit":0.003091573715209961,"oracle_top2_margin":0.9089595079421997,"near_tie":false,"probs":[0.025944460183382034,0.03241077437996864,0.9416447877883911]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00016760826110839844,"mean_abs_dp":0.00011171400547027588,"max_abs_dlogit":0.003091573715209961},{"id":"semif_942ec65dac0bfb9874f9","arm":"text","variant":"base","tokens":177,"calls":3,"padded_tokens":192,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.42950079101137817,"wall_seconds":0.4406976659956854,"finite":true,"index":34,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":1.0411415100097656,"ref_absmax":48.31693649291992,"rel_max_abs_diff":0.02154816893580015,"min_pos_cos":0.9993045590047477,"min_pos_cos_index":72,"mean_pos_cos":0.9999621755331486,"positions":177,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.00041538476943969727,"mean_abs_dp":0.0002769088993469874,"max_abs_dlogit":0.004327058792114258,"oracle_top2_margin":0.7890182957053185,"near_tie":false,"probs":[0.09143523871898651,0.8796609044075012,0.02890387736260891]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00041538476943969727,"mean_abs_dp":0.0002769088993469874,"max_abs_dlogit":0.004327058792114258},{"id":"semif_e0c140e9222d1bce5647","arm":"text","variant":"base","tokens":189,"calls":3,"padded_tokens":192,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.42876091599464417,"wall_seconds":0.4402651670097839,"finite":true,"index":35,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":0.8863525390625,"ref_absmax":48.31690979003906,"rel_max_abs_diff":0.01834456182968119,"min_pos_cos":0.9981196130586676,"min_pos_cos_index":66,"mean_pos_cos":0.9999611026537827,"positions":189,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00011587142944335938,"mean_abs_dp":7.723147670427959e-05,"max_abs_dlogit":0.002429962158203125,"oracle_top2_margin":0.8911582641303539,"near_tie":false,"probs":[0.03848356381058693,0.03203703463077545,0.9294793605804443]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00011587142944335938,"mean_abs_dp":7.723147670427959e-05,"max_abs_dlogit":0.002429962158203125},{"id":"semif_8e8c3804a3c15ebb31e3","arm":"text","variant":"base","tokens":186,"calls":3,"padded_tokens":192,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.4290982079983223,"wall_seconds":0.4401374169974588,"finite":true,"index":36,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":1.1604747772216797,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.02401801077437866,"min_pos_cos":0.9981200757558389,"min_pos_cos_index":66,"mean_pos_cos":0.999942426092925,"positions":186,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.0014159530401229858,"mean_abs_dp":0.0009439804901679357,"max_abs_dlogit":0.015268325805664062,"oracle_top2_margin":0.7645951732993126,"near_tie":false,"probs":[0.8710676431655884,0.019662655889987946,0.10926970094442368]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0014159530401229858,"mean_abs_dp":0.0009439804901679357,"max_abs_dlogit":0.015268325805664062},{"id":"semif_56c419b12cd37cbe790c","arm":"text","variant":"base","tokens":188,"calls":3,"padded_tokens":192,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.4289383330033161,"wall_seconds":0.4398521250113845,"finite":true,"index":37,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":1.2368364334106445,"ref_absmax":48.316871643066406,"rel_max_abs_diff":0.02559843779927614,"min_pos_cos":0.9978113761327639,"min_pos_cos_index":65,"mean_pos_cos":0.9999323945679409,"positions":188,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.0001563429832458496,"mean_abs_dp":0.00010426218310991923,"max_abs_dlogit":0.0027655363082885742,"oracle_top2_margin":0.9058033861219883,"near_tie":false,"probs":[0.9396253824234009,0.02630753442645073,0.03406718000769615]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0001563429832458496,"mean_abs_dp":0.00010426218310991923,"max_abs_dlogit":0.0027655363082885742}],"compare_with":{"other":"<lane>/results/readout_fp16_pf64_full.json","other_chunk":64,"other_bundle":"clef_flash_decode_fp16_pf64","runs":38,"max_abs_dp":0.0026382803916931152,"argmax_equal_runs":38,"hidden_max_abs_diff":5.53125,"vs_oracle_on_shared_runs":{"order":["other","this"],"max_abs_dp":[0.007522463798522949,0.007514894008636475],"min_pos_cos":[0.4074876250032938,0.37844254882577144],"positions_below_0.99":[48,53]},"timing_warm_common_runs":{"other":{"S":64,"runs":36,"ms_per_call_median":134.4399164954666,"ms_per_token_median":2.1885271943766935,"run_decode_seconds_median":1.0755103750125272,"calls_total":382},"this":{"S":64,"runs":36,"ms_per_call_median":143.51289549085777,"ms_per_token_median":2.3438459856447382,"run_decode_seconds_median":1.1463873534958111,"calls_total":382}}},"note":"round 4 step 4 diagnostic (not a ship form): the 8 full-attention layers' q/k/v/o linears and SDPA in fp32 (fp16 weights, fp32 compute); 38 runs = the 29 low-cos runs of pf16 + own_t01/own_j01/img_01 g256/img_04 g448 (2 already among the 29) + the first 7 SemIf","generated_at":"2026-10-03T04:12:58+00:00"}},"fp16_pf128":{"runs_file":{"runs":[["own_t01","text"],["own_t02","text"],["own_t03","text"],["own_t04","text"],["own_t05","text"],["own_t14","text"],["own_t15","text"],["own_j01","text"],["own_j02","text"],["own_j03","text"],["own_j04","text"],["img_01","g448"],["img_02","g448"],["img_03","g448"],["img_04","g448"],["semif_a3f18f3a63d45345942b","text"],["semif_f40beba9088c8db8bbd6","text"],["semif_0b43ea8e24e74d621f1b","text"],["semif_942ec65dac0bfb9874f9","text"],["semif_e0c140e9222d1bce5647","text"]],"why":"round 4 step 5 (S=128 trial): the first 5 own_text, 2 own_text_long, 4 own_json, 4 own_image g448 and 5 SemIf runs in oracle order (launch step 5: own_text 5, long 2, json 4, img g448 4, semif 5)"},"export_record":{"mode":"fp16","name":"clef_flash_decode_fp16_pf128","hf_id":"Cloudflare/clef-flash","revision":"17f0b0ad64efb65d273590632833508766b2aae6","prefill_chunk":128,"argv":["fp16","--prefill-chunk","128","--aot","--record","<lane>/logs/r4_export_fp16_pf128.json"],"pid":48233,"started":"2026-10-03T13:13:31+09:00","script_sha256":"fbc1d712842357bd57735f9acf7d47042f14f966d7c4ca5c5a358db3cfbd5a68","module_sha256":"6342008b9fde0a521aa4db5b7ac9e164c93086a7cae92ee19ba1c2e356f59b8f","parent_sha256":"7f7fd56967f7ba4ca9e9ca12092205df638d6996470454cfc0845b495090211f","export":{"bundle":"<lane>/exports/bundles/clef_flash_decode_fp16_pf128","name":"clef_flash_decode_fp16_pf128","aimodel":"<lane>/exports/bundles/clef_flash_decode_fp16_pf128/clef_flash_decode_fp16_pf128.aimodel","load_report":{"checkpoint_text_keys":427,"checkpoint_other_keys_skipped":455,"module_tensors":426,"unread_checkpoint_keys":[],"module_tensors_not_in_checkpoint":[],"tied_lm_head":false,"intentionally_unread_keys":["lm_head.weight"],"lm_head_untied":true,"checkpoint_lm_head":{"file":"model-00001-of-00004.safetensors","dtype":"BF16","shape":[248320,4096]},"module_has_lm_head_weight":false},"loopfree_unrolled_linear_layers":24,"trace_kv_len":2048,"max_ctx":4096,"n_image_max":1024,"functions":["main"],"query_len":128,"gdn_scan":"unroll","output_names":["hidden"],"quantization":null,"variant":null,"seconds":{"load":6.2558642080111895,"quantize":0.06630329199833795,"export":162.2943929579924,"optimize":2.3529320000088774,"export_optimize":164.64732495800126,"save":11.89258862499264,"total":182.9666786660091},"du_aimodel":"15G","du_bundle":"15G","aimodel_files":{"main.hash":32,"main.mlirb":15885881371,"metadata.json":105},"main_mlirb":{"bytes":15885881371,"sha256":"c65c696d905a9f0a71aab7f2512d26de459abcd2cca990e7c74214bf456a1685"},"metadata_sha256":"7d134c062326218720ca123b429d72f3c0f022821924a660289f95f08780dd6b","kind":"decision-backbone","tokenizer_sha256":{"chat_template.jinja":"a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715","tokenizer.json":"06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523","tokenizer_config.json":"91a08f825d370d085d692e04cf117cdd7faad7bf18e996f1e6031b6dab03db72"},"tokenizer_verbatim_from_snapshot":true,"disk":{"before_load":{"time":"2026-10-03T13:13:33+09:00","free_bytes":158694178816,"free_gib":147.8,"caches":{"coreai_cache_python":"32G","mpsgraph_scratch":{"/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph":"0B"}}},"after_load":{"time":"2026-10-03T13:13:40+09:00","free_bytes":158692024320,"free_gib":147.8,"caches":{"coreai_cache_python":"32G","mpsgraph_scratch":{"/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph":"0B"}}},"after_export":{"time":"2026-10-03T13:16:24+09:00","free_bytes":142965530624,"free_gib":133.1,"caches":{"coreai_cache_python":"32G","mpsgraph_scratch":{"/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph":"0B"}}},"after_save":{"time":"2026-10-03T13:16:36+09:00","free_bytes":127065858048,"free_gib":118.3,"caches":{"coreai_cache_python":"32G","mpsgraph_scratch":{"/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph":"0B"}}}}},"aot":{"aimodelc":"<lane>/exports/bundles_aotc/clef_flash_decode_fp16_pf128.h16c.aimodelc","flags":["--platform","macOS","--preferred-compute","gpu","--architecture","h16c","--expect-frequent-reshapes"],"seconds":130.81543604200124,"coreai_build":"/var/run/com.apple.security.cryptexd/mnt/com.apple.MobileAsset.MetalToolchain-v27.1.266.1.6t2kcX/Metal.xctoolchain/usr/bin/coreai-build","command":["/var/run/com.apple.security.cryptexd/mnt/com.apple.MobileAsset.MetalToolchain-v27.1.266.1.6t2kcX/Metal.xctoolchain/usr/bin/coreai-build","compile","<lane>/exports/bundles/clef_flash_decode_fp16_pf128/clef_flash_decode_fp16_pf128.aimodel","--output","<lane>/exports/bundles_aotc","--platform","macOS","--preferred-compute","gpu","--architecture","h16c","--expect-frequent-reshapes"],"disk":{"before":{"time":"2026-10-03T13:16:42+09:00","free_bytes":127045787648,"free_gib":118.3,"caches":{"coreai_cache_python":"32G","mpsgraph_scratch":{"/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph":"0B"}}},"after":{"time":"2026-10-03T13:18:53+09:00","free_bytes":92998742016,"free_gib":86.6,"caches":{"coreai_cache_python":"32G","mpsgraph_scratch":{"/private/var/folders/pn/y692nwgx6_s29gl7m57dhtkh0000gn/T/com.apple.MetalPerformanceShadersGraph":"0B"}}}},"du_aimodelc":"28G","digest":{"bytes":29750350340,"tree_sha256":"68be50054027838760d79d57f0fba1cfd2b28a05ae55d7213caaf355bcd276af","files":{"main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/manifest.plist":"cb1b39a46b3ad3094c80a76d5284602adec99e7f93597fb8729240dce278d4fe","main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/original_model_0.mpsgraph":"c4ae34b9edf15359878641ce89f382dc46b9196b5550ac20f29d75591669ba71","main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/resources.bin":"c8cd79eada1a2a305fa175a5234c72be909f46bffaf0f55dfcf1590dcaedcf59","main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/specialized_model_1.mpsgraph":"3bedf4418295801184aa5c49c8d1bef19c69b51a5bbaa46ff57881e6004ef08c","main-h16c.mlirb":"32aa5498a264383bf57c601309248db7acf2380177c5e544bd3b6fd9aca7f63c","main.hash":"11fbcd5ebfbf8dae454ffb6ab18343e0449416401a616c1a5b97dc8a5bc02fc4","metadata.json":"1924b69f54163f47bf9716da63a036ae333b3695a7b8d23096b0aac4ea310cd0","stats.json":"8a2c0b217fe3221086ef2adbf64124e8c98c09fe826e04cf4afc0adb3e0c508e"}}},"finished":"2026-10-03T13:19:03+09:00"},"readout":{"schema":"clef-flash-decoder-readout-gate/1","gate":"decoder alone on the Mac GPU (oracle ids, oracle fp32 image_embeds -> fp16, zero-padded to 1024 rows), the fp16 hidden read through the author's fp32 head","bundle":{"bundle":"<lane>/exports/bundles/clef_flash_decode_fp16_pf128","name":"clef_flash_decode_fp16_pf128","kind":"decision-backbone","compression":null,"prefill_chunk":128,"metadata_sha256":"7d134c062326218720ca123b429d72f3c0f022821924a660289f95f08780dd6b","main_mlirb":{"bytes":15885881371,"sha256":"c65c696d905a9f0a71aab7f2512d26de459abcd2cca990e7c74214bf456a1685"},"tokenizer_sha256":{"chat_template.jinja":"a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715","tokenizer.json":"06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523","tokenizer_config.json":"91a08f825d370d085d692e04cf117cdd7faad7bf18e996f1e6031b6dab03db72"},"aimodelc":{"path":"<lane>/exports/bundles_aotc/clef_flash_decode_fp16_pf128.h16c.aimodelc","bytes":29750350340,"tree_sha256":"68be50054027838760d79d57f0fba1cfd2b28a05ae55d7213caaf355bcd276af","files":{"main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/manifest.plist":"cb1b39a46b3ad3094c80a76d5284602adec99e7f93597fb8729240dce278d4fe","main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/original_model_0.mpsgraph":"c4ae34b9edf15359878641ce89f382dc46b9196b5550ac20f29d75591669ba71","main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/resources.bin":"c8cd79eada1a2a305fa175a5234c72be909f46bffaf0f55dfcf1590dcaedcf59","main-h16c-delegates/MPSGraph/mpsExecutable.mpsgraphpackage/specialized_model_1.mpsgraph":"3bedf4418295801184aa5c49c8d1bef19c69b51a5bbaa46ff57881e6004ef08c","main-h16c.mlirb":"32aa5498a264383bf57c601309248db7acf2380177c5e544bd3b6fd9aca7f63c","main.hash":"11fbcd5ebfbf8dae454ffb6ab18343e0449416401a616c1a5b97dc8a5bc02fc4","metadata.json":"1924b69f54163f47bf9716da63a036ae333b3695a7b8d23096b0aac4ea310cd0","stats.json":"8a2c0b217fe3221086ef2adbf64124e8c98c09fe826e04cf4afc0adb3e0c508e"}}},"chunk":128,"max_ctx":4096,"readout":"fresh zero states; a row of T ids runs as ceil(T / 128) calls of 'main' (static S = 128): call k gets ids[128k : 128k + 128] with position_ids 0..128k+127; the last call is padded with <|endoftext|> (248044) and the hidden rows of the padded positions are discarded (causal: they cannot reach a real position). The hidden rows [1, 128, 4096] of every call, concatenated and cut to T, are the backbone's final-norm last_hidden_state [T, 4096] the head reads.","descriptor":{"function":"main","inputs":{"input_ids":[[1,128],"int32"],"position_ids":[[1,-1],"int32"],"image_embeds":[[1024,4096],"float16"],"image_rc":[[1024,2],"int32"],"rope_shift_start":[[1],"int32"],"rope_shift_amount":[[1],"int32"]},"outputs":{"hidden":[[1,128,4096],"float16"]},"states":{"keyCache":[[8,1,4,-1,256],"float16"],"valueCache":[[8,1,4,-1,256],"float16"],"convState":[[24,1,8192,3],"float16"],"recState":[[24,1,32,128,128],"float16"]}},"descriptor_same_in_every_process":true,"contract":{"inputs":{"input_ids":[[1,128],"int32"],"position_ids":[[1,-1],"int32"],"image_embeds":[[1024,4096],"float16"],"image_rc":[[1024,2],"int32"],"rope_shift_start":[[1],"int32"],"rope_shift_amount":[[1],"int32"]},"outputs":{"hidden":[[1,128,4096],"float16"]},"states":{"keyCache":[[8,1,4,-1,256],"float16"],"valueCache":[[8,1,4,-1,256],"float16"],"convState":[[24,1,8192,3],"float16"],"recState":[[24,1,32,128,128],"float16"]}},"oracle":{"path":"<lane>/oracle/records_oracle.json","sha256":"d8adaa746a5d575e654212ca78f5634af334aaa62eb53ef150d5d5925b1fa7c0","source":"Cloudflare/clef-flash@17f0b0ad64efb65d273590632833508766b2aae6","versions":{"python":"3.12.11","torch":"2.9.0","torchvision":"0.24.0","transformers":"5.17.0","pillow":"12.3.0","numpy":"2.5.3","huggingface_hub":"1.33.0"}},"subset":"list","runs_file":{"path":"<lane>/results/r4_runs_s128.json","sha256":"fdc9235cbefef4181b266d8a704a2cdb235545e72e66cc392414d9664447a0d9","why":"round 4 step 5 (S=128 trial): the first 5 own_text, 2 own_text_long, 4 own_json, 4 own_image g448 and 5 SemIf runs in oracle order (launch step 5: own_text 5, long 2, json 4, img g448 4, semif 5)"},"script":{"path":"conversion/clef_flash/readout_gate.py","sha256":"43727fedef59375e6f4c77d3a32d9623659ea6f71ad8948bb439c06c11928006"},"skipped":[{"run":"photo_01/native","reason":"N = 40 x 30 = 1,200 image rows > the graph's 1024-row buffer (outside the fixed-grid path), not run","questions":4}],"bar":{"max_abs_dp":0.02,"mean_of_run_mean_abs_dp":0.002,"near_tie_top2_margin":0.02,"argmax":"every question with an oracle top-2 margin above 0.02; near-ties listed apart","max_abs_dp_applies_to":"every option of every question, near-ties included","reset":"bit-equal hidden rows in every process","mean_of_run_mean_abs_dp_definition":"mean over runs of the mean |dp| over all (question, option) entries of the run"},"environment":{"versions":{"python":"3.11.13","numpy":"2.3.5","coreai-core":"1.0.0b2","coreai-torch":"0.4.1","coreai-models":"0.1.0","torch":"2.9.0","safetensors":"0.7.0"},"platform":"macOS-27.0-arm64-arm-64bit","macos_build":"26A428","chip":"Apple M4 Max","runtime":"coreai python runtime, AOT h16c GPU .aimodelc, SpecializationOptions.default(), no JIT","head":"the author's JointSchemaHead, fp32, CPU (parity_decoder_torch.AuthorHead)","gpu":"shared with other sessions, no _GPU_LOCK held (times are contended reference values)"},"other_gpu_processes":{"note":"process listing removed before publication (4 lines); it showed other sessions' GPU work at the time = the times are contended"},"processes":[{"shard":"shard_00","runs":["own_t01/text/base","own_t02/text/base","own_t03/text/base","own_t04/text/base","own_t05/text/base","own_t14/text/base","own_t15/text/base","own_j01/text/base","own_j02/text/base","own_j03/text/base","own_j04/text/base","img_01/g448/base","img_02/g448/base","img_03/g448/base","img_04/g448/base","semif_a3f18f3a63d45345942b/text/base","semif_f40beba9088c8db8bbd6/text/base","semif_0b43ea8e24e74d621f1b/text/base","semif_942ec65dac0bfb9874f9/text/base","semif_e0c140e9222d1bce5647/text/base"],"prompts":21,"pid":52652,"load_seconds":24.684863458998734,"load_split_seconds":{"model":5.51856399999815,"main":19.166299459000584},"function_names":["main"],"process_wall_seconds":81.76896666598623,"reset_check":{"run":["own_t01","text","base"],"bit_equal":true,"hidden_max_abs_diff":0.0,"wall_seconds":0.9939137090113945},"contract_mismatch":[]}],"summary":{"runs":20,"expected_runs":20,"missing_runs":[],"questions":61,"argmax_equal":61,"questions_non_near_tie":61,"argmax_equal_non_near_tie":61,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.007786452770233154,"max_abs_dp_non_near_tie":0.007786452770233154,"mean_of_run_mean_abs_dp":0.00039211717058659155,"mean_question_max_abs_dp":0.000623592595401846,"max_abs_dlogit":0.03336942195892334,"min_pos_cos":0.9479832238632915,"positions":13029,"positions_below":{"0.99":6,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":2,"runs_with_positions_below_0.99":["own_t01/text","own_t04/text","own_t15/text","img_04/g448"],"hidden_max_abs_diff":3.5154056549072266,"hidden_max_rel_diff":0.07275733394052443,"worst_run":{"id":"own_t03","arm":"text","max_abs_dp":0.007786452770233154},"finite_all":true,"tokens":13029,"calls":111,"ms_per_call_median":246.26345798606053,"ms_per_call_warm_median":246.25054199714214},"by_source_arm":{"own_image/g448":{"runs":4,"expected_runs":4,"missing_runs":[],"questions":13,"argmax_equal":13,"questions_non_near_tie":13,"argmax_equal_non_near_tie":13,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.0007541775703430176,"max_abs_dp_non_near_tie":0.0007541775703430176,"mean_of_run_mean_abs_dp":0.00020064083280775584,"mean_question_max_abs_dp":0.0003783043760519761,"max_abs_dlogit":0.01517033576965332,"min_pos_cos":0.98531842070341,"positions":2477,"positions_below":{"0.99":2,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":2,"runs_with_positions_below_0.99":["img_04/g448"],"hidden_max_abs_diff":3.210784912109375,"hidden_max_rel_diff":0.06645268654427998,"worst_run":{"id":"img_01","arm":"g448","max_abs_dp":0.0007541775703430176},"finite_all":true,"tokens":2477,"calls":21,"ms_per_call_median":245.85875001503155,"ms_per_call_warm_median":245.85875001503155},"own_json/text":{"runs":4,"expected_runs":4,"missing_runs":[],"questions":15,"argmax_equal":15,"questions_non_near_tie":15,"argmax_equal_non_near_tie":15,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.007267773151397705,"max_abs_dp_non_near_tie":0.007267773151397705,"mean_of_run_mean_abs_dp":0.000531833572010812,"mean_question_max_abs_dp":0.0009039908647537232,"max_abs_dlogit":0.02937138080596924,"min_pos_cos":0.9927338004760681,"positions":2105,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0,"runs_with_positions_below_0.99":[],"hidden_max_abs_diff":2.564862087368965,"hidden_max_rel_diff":0.05308411229145641,"worst_run":{"id":"own_j04","arm":"text","max_abs_dp":0.007267773151397705},"finite_all":true,"tokens":2105,"calls":18,"ms_per_call_median":245.7330625038594,"ms_per_call_warm_median":245.7330625038594},"own_text/text":{"runs":5,"expected_runs":5,"missing_runs":[],"questions":18,"argmax_equal":18,"questions_non_near_tie":18,"argmax_equal_non_near_tie":18,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.007786452770233154,"max_abs_dp_non_near_tie":0.007786452770233154,"mean_of_run_mean_abs_dp":0.0008380265391760178,"mean_question_max_abs_dp":0.0009165406485812531,"max_abs_dlogit":0.03336942195892334,"min_pos_cos":0.9780664300848105,"positions":2544,"positions_below":{"0.99":2,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0,"runs_with_positions_below_0.99":["own_t01/text","own_t04/text"],"hidden_max_abs_diff":3.5154056549072266,"hidden_max_rel_diff":0.07275733394052443,"worst_run":{"id":"own_t03","arm":"text","max_abs_dp":0.007786452770233154},"finite_all":true,"tokens":2544,"calls":22,"ms_per_call_median":246.2687920051394,"ms_per_call_warm_median":246.18683350854553},"own_text_long/text":{"runs":2,"expected_runs":2,"missing_runs":[],"questions":10,"argmax_equal":10,"questions_non_near_tie":10,"argmax_equal_non_near_tie":10,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.000980973243713379,"max_abs_dp_non_near_tie":0.000980973243713379,"mean_of_run_mean_abs_dp":0.00014640877418445818,"mean_question_max_abs_dp":0.00024193190038204193,"max_abs_dlogit":0.01731729507446289,"min_pos_cos":0.9479832238632915,"positions":4992,"positions_below":{"0.99":2,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0,"runs_with_positions_below_0.99":["own_t15/text"],"hidden_max_abs_diff":3.4329700469970703,"hidden_max_rel_diff":0.06425275932752302,"worst_run":{"id":"own_t14","arm":"text","max_abs_dp":0.000980973243713379},"finite_all":true,"tokens":4992,"calls":40,"ms_per_call_median":247.51243750506546,"ms_per_call_warm_median":247.51243750506546},"semif_authored144/text":{"runs":5,"expected_runs":5,"missing_runs":[],"questions":5,"argmax_equal":5,"questions_non_near_tie":5,"argmax_equal_non_near_tie":5,"near_tie_questions":0,"argmax_equal_near_tie":0,"max_abs_dp":0.00024896860122680664,"max_abs_dp_non_near_tie":0.00024896860122680664,"mean_of_run_mean_abs_dp":8.589910964171092e-05,"mean_question_max_abs_dp":0.00012885555624961854,"max_abs_dlogit":0.003228425979614258,"min_pos_cos":0.9985240924527983,"positions":911,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0,"runs_with_positions_below_0.99":[],"hidden_max_abs_diff":1.2367324829101562,"hidden_max_rel_diff":0.025596245949194053,"worst_run":{"id":"semif_942ec65dac0bfb9874f9","arm":"text","max_abs_dp":0.00024896860122680664},"finite_all":true,"tokens":911,"calls":10,"ms_per_call_median":245.60937499336433,"ms_per_call_warm_median":245.60937499336433}},"near_ties":[],"checks":{"all_runs":true,"argmax_non_near_tie":true,"max_abs_dp":true,"mean_of_run_mean_abs_dp":true,"finite":true,"reset_bit_equal_all_processes":true},"result":"PASS","red_arm":null,"timing":{"contended":true,"S":128,"load_seconds":{"first_process":24.684863458998734,"median":24.684863458998734,"all":[24.684863458998734]},"ms_per_call_warm_runs":{"n":107,"median":246.25054199714214,"p10":245.10737520176917,"p90":248.48281679442152},"ms_per_call_all_runs":{"n":111,"median":246.26345798606053,"p10":245.12079200940207,"p90":248.57141700340435},"first_call_of_process_ms":[26021.878083003685],"ms_per_token_warm_runs":{"n":19,"median":2.13692451736964,"p10":1.9887477785887178,"p90":2.70017820637374},"run_decode_seconds_warm_runs":{"n":19,"median":1.2272143329901155,"p10":0.49093417478725315,"p90":2.119957933586554},"run_wall_seconds_warm_runs":{"n":19,"median":1.2401640829921234,"p10":0.5018545754079241,"p90":2.13803398341988},"by_source_arm_warm":{"own_image/g448":{"runs":4,"median_tokens":612.5,"median_calls":5.0,"ms_per_call_median":245.85875001503155,"run_decode_seconds_median":1.2283904585056007},"own_json/text":{"runs":4,"median_tokens":522.5,"median_calls":4.5,"ms_per_call_median":245.7330625038594,"run_decode_seconds_median":1.106546458991943},"own_text/text":{"runs":4,"median_tokens":523.0,"median_calls":4.5,"ms_per_call_median":246.18683350854553,"run_decode_seconds_median":1.1117704170028446},"own_text_long/text":{"runs":2,"median_tokens":2496.0,"median_calls":20.0,"ms_per_call_median":247.51243750506546,"run_decode_seconds_median":4.951707041996997},"semif_authored144/text":{"runs":5,"median_tokens":183.0,"median_calls":2.0,"ms_per_call_median":245.60937499336433,"run_decode_seconds_median":0.4909775829873979}}},"seconds":{"gpu_processes":81.80122783398838,"scoring":19.097516415989958,"total":100.98285345899058},"runs":[{"id":"own_t01","arm":"text","variant":"base","tokens":436,"calls":4,"padded_tokens":512,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":26.760400625003967,"wall_seconds":26.773789583006874,"finite":true,"index":0,"shard":"shard_00","first_in_process":true,"source":"own_text","hidden":{"max_abs_diff":3.5154056549072266,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.07275733394052443,"min_pos_cos":0.9849953430451334,"min_pos_cos_index":402,"mean_pos_cos":0.9999318434406395,"positions":436,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"department","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":7.361173629760742e-05,"mean_abs_dp":3.678642679005861e-05,"max_abs_dlogit":0.0037065744400024414,"oracle_top2_margin":0.9682950582355261,"near_tie":false,"probs":[0.0020665430929511786,0.009686468169093132,0.009927275590598583,0.9783197045326233]},{"question_id":"wants_refund","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":2.008676528930664e-05,"mean_abs_dp":2.0041130483150482e-05,"max_abs_dlogit":0.0038878917694091797,"oracle_top2_margin":0.9727394469082355,"near_tie":false,"probs":[0.013650277629494667,0.9863496422767639]},{"question_id":"urgency","type":"score","n_options":4,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0017347335815429688,"mean_abs_dp":0.0008673854172229767,"max_abs_dlogit":0.009391546249389648,"oracle_top2_margin":0.07407951354980469,"near_tie":false,"probs":[0.3086758553981781,0.3806334137916565,0.22407560050487518,0.08661504834890366]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0017347335815429688,"mean_abs_dp":0.00036567696370184424,"max_abs_dlogit":0.009391546249389648},{"id":"own_t02","arm":"text","variant":"base","tokens":599,"calls":5,"padded_tokens":640,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":1.2298528339888435,"wall_seconds":1.243861791008385,"finite":true,"index":1,"shard":"shard_00","first_in_process":false,"source":"own_text","hidden":{"max_abs_diff":1.444854736328125,"ref_absmax":51.46957778930664,"rel_max_abs_diff":0.02807201454503303,"min_pos_cos":0.9983114903390506,"min_pos_cos_index":561,"mean_pos_cos":0.9999673553741283,"positions":599,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"severity","type":"score","n_options":5,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0014151930809020996,"mean_abs_dp":0.0005660872906446457,"max_abs_dlogit":0.004918098449707031,"oracle_top2_margin":0.1987611949443817,"near_tie":false,"probs":[0.030549228191375732,0.16500312089920044,0.47725433111190796,0.27656689286231995,0.05062638968229294]},{"question_id":"resolved","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.00022574886679649353,"mean_abs_dp":0.00022573582828044891,"max_abs_dlogit":0.004927396774291992,"oracle_top2_margin":0.9355993382632732,"near_tie":false,"probs":[0.9680253863334656,0.03197457641363144]},{"question_id":"root_cause","type":"choice","n_options":5,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":9.362585842609406e-05,"mean_abs_dp":3.745108842849732e-05,"max_abs_dlogit":0.006892681121826172,"oracle_top2_margin":0.9514973126351833,"near_tie":false,"probs":[0.001320433453656733,0.02153407596051693,0.9728679060935974,0.0018683337839320302,0.002409223932772875]},{"question_id":"data_loss","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":9.18237492442131e-06,"mean_abs_dp":9.180745109915733e-06,"max_abs_dlogit":0.004393577575683594,"oracle_top2_margin":0.9881136221811175,"near_tie":false,"probs":[0.005934016313403845,0.9940659999847412]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0014151930809020996,"mean_abs_dp":0.0002491089315818889,"max_abs_dlogit":0.006892681121826172},{"id":"own_t03","arm":"text","variant":"base","tokens":463,"calls":4,"padded_tokens":512,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.9936880000168458,"wall_seconds":1.0044385000073817,"finite":true,"index":2,"shard":"shard_00","first_in_process":false,"source":"own_text","hidden":{"max_abs_diff":1.4808197021484375,"ref_absmax":56.47786331176758,"rel_max_abs_diff":0.026219471051411,"min_pos_cos":0.9906556671092915,"min_pos_cos_index":302,"mean_pos_cos":0.9998974353971865,"positions":463,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"new_slot","type":"choice","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0007303953170776367,"mean_abs_dp":0.00048695287356774014,"max_abs_dlogit":0.010651826858520508,"oracle_top2_margin":0.8634682968258858,"near_tie":false,"probs":[0.012107913382351398,0.06150683015584946,0.926385223865509]},{"question_id":"thursday_still_on","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.003393813967704773,"mean_abs_dp":0.0033937618136405945,"max_abs_dlogit":0.03336942195892334,"oracle_top2_margin":0.8205529078841209,"near_tie":false,"probs":[0.08632975071668625,0.9136701822280884]},{"question_id":"reply_needed","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.007786452770233154,"mean_abs_dp":0.0077864378690719604,"max_abs_dlogit":0.027570724487304688,"oracle_top2_margin":0.5112259685993195,"near_tie":false,"probs":[0.7478265166282654,0.25217342376708984]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.007786452770233154,"mean_abs_dp":0.00340303685516119,"max_abs_dlogit":0.03336942195892334},{"id":"own_t04","arm":"text","variant":"base","tokens":493,"calls":4,"padded_tokens":512,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.9829930000123568,"wall_seconds":0.9937716670101508,"finite":true,"index":3,"shard":"shard_00","first_in_process":false,"source":"own_text","hidden":{"max_abs_diff":2.014384388923645,"ref_absmax":48.97331619262695,"rel_max_abs_diff":0.04113228479363044,"min_pos_cos":0.9780664300848105,"min_pos_cos_index":113,"mean_pos_cos":0.9998845860009997,"positions":493,"positions_below":{"0.99":1,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"dark_mode","type":"choice","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00011068582534790039,"mean_abs_dp":7.381134976943333e-05,"max_abs_dlogit":0.007947921752929688,"oracle_top2_margin":0.9783774577081203,"near_tie":false,"probs":[0.00723820086568594,0.007273099850863218,0.9854887127876282]},{"question_id":"release_note_owner","type":"choice","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.00015610456466674805,"mean_abs_dp":0.00010407809168100357,"max_abs_dlogit":0.008428901433944702,"oracle_top2_margin":0.9677823921665549,"near_tie":false,"probs":[0.0114901103079319,0.9790094494819641,0.00950042437762022]},{"question_id":"open_items","type":"score","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":3.36766242980957e-05,"mean_abs_dp":2.2488801429669063e-05,"max_abs_dlogit":0.003036022186279297,"oracle_top2_margin":0.9687048075720668,"near_tie":false,"probs":[0.0120931351557374,0.9807443022727966,0.007162646856158972]},{"question_id":"beta_label_decided","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0002774633467197418,"mean_abs_dp":0.0002774316817522049,"max_abs_dlogit":0.012467145919799805,"oracle_top2_margin":0.9487586636096239,"near_tie":false,"probs":[0.025898100808262825,0.9741019010543823]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0002774633467197418,"mean_abs_dp":0.0001050907356495207,"max_abs_dlogit":0.012467145919799805},{"id":"own_t05","arm":"text","variant":"base","tokens":553,"calls":5,"padded_tokens":640,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":1.2305520420195535,"wall_seconds":1.241305416013347,"finite":true,"index":4,"shard":"shard_00","first_in_process":false,"source":"own_text","hidden":{"max_abs_diff":2.1529693603515625,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.04455938406316009,"min_pos_cos":0.9954402448266317,"min_pos_cos_index":314,"mean_pos_cos":0.9999631230104518,"positions":553,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"sentiment","type":"score","n_options":5,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0002701282501220703,"mean_abs_dp":0.00010806974023580551,"max_abs_dlogit":0.005004525184631348,"oracle_top2_margin":0.28853923082351685,"near_tie":false,"probs":[0.02908516116440296,0.6238692402839661,0.3357457220554352,0.008223933167755604,0.003075925400480628]},{"question_id":"safety_issue","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":7.56978988647461e-05,"mean_abs_dp":7.566716521978378e-05,"max_abs_dlogit":0.0038353800773620605,"oracle_top2_margin":0.9395809434354305,"near_tie":false,"probs":[0.969866156578064,0.03013387881219387]},{"question_id":"main_complaint","type":"choice","n_options":4,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":4.934798926115036e-05,"mean_abs_dp":2.466142177581787e-05,"max_abs_dlogit":0.006815910339355469,"oracle_top2_margin":0.9631505683064461,"near_tie":false,"probs":[0.006244494114071131,0.9740009903907776,0.008973687887191772,0.010780868120491505]},{"question_id":"recommend","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":4.178285598754883e-05,"mean_abs_dp":4.176050424575806e-05,"max_abs_dlogit":0.0016442537307739258,"oracle_top2_margin":0.9594815410673618,"near_tie":false,"probs":[0.02021745964884758,0.9797825217247009]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0002701282501220703,"mean_abs_dp":6.721920978564482e-05,"max_abs_dlogit":0.006815910339355469},{"id":"own_t14","arm":"text","variant":"base","tokens":2389,"calls":19,"padded_tokens":2432,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":4.7018779999925755,"wall_seconds":4.71290641700034,"finite":true,"index":5,"shard":"shard_00","first_in_process":false,"source":"own_text_long","hidden":{"max_abs_diff":1.1966896057128906,"ref_absmax":48.31694412231445,"rel_max_abs_diff":0.024767493628807903,"min_pos_cos":0.9902812624598238,"min_pos_cos_index":1532,"mean_pos_cos":0.9999511073472622,"positions":2389,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"first_failing_component","type":"choice","n_options":5,"argmax":4,"argmax_oracle":4,"argmax_equal":true,"max_abs_dp":0.00036346912384033203,"mean_abs_dp":0.00014535794034600257,"max_abs_dlogit":0.01731729507446289,"oracle_top2_margin":0.9446960557252169,"near_tie":false,"probs":[0.01681128516793251,0.006257630418986082,0.005260748788714409,0.009470287710428238,0.9622001647949219]},{"question_id":"recovered","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.0001474618911743164,"mean_abs_dp":0.00014743953943252563,"max_abs_dlogit":0.00608372688293457,"oracle_top2_margin":0.9060924127697945,"near_tie":false,"probs":[0.9528987407684326,0.047101207077503204]},{"question_id":"severity","type":"score","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00015783309936523438,"mean_abs_dp":8.460972458124161e-05,"max_abs_dlogit":0.007518768310546875,"oracle_top2_margin":0.6201646775007248,"near_tie":false,"probs":[0.031874168664216995,0.06415053457021713,0.7619210481643677,0.1420542150735855]},{"question_id":"error_window","type":"choice","n_options":4,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.000980973243713379,"mean_abs_dp":0.0004946282133460045,"max_abs_dlogit":0.013701438903808594,"oracle_top2_margin":0.6222178637981415,"near_tie":false,"probs":[0.1637383997440338,0.7876051664352417,0.04060051217675209,0.008055882528424263]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.000980973243713379,"mean_abs_dp":0.00022257470215360323,"max_abs_dlogit":0.01731729507446289},{"id":"own_t15","arm":"text","variant":"base","tokens":2603,"calls":21,"padded_tokens":2688,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":5.201536084001418,"wall_seconds":5.211483874998521,"finite":true,"index":6,"shard":"shard_00","first_in_process":false,"source":"own_text_long","hidden":{"max_abs_diff":3.4329700469970703,"ref_absmax":53.42914581298828,"rel_max_abs_diff":0.06425275932752302,"min_pos_cos":0.9479832238632915,"min_pos_cos_index":2530,"mean_pos_cos":0.9999250184656937,"positions":2603,"positions_below":{"0.99":2,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"non_renewal_notice","type":"choice","n_options":5,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":1.609325408935547e-05,"mean_abs_dp":7.697008550167084e-06,"max_abs_dlogit":0.004372358322143555,"oracle_top2_margin":0.9844898809678853,"near_tie":false,"probs":[0.004058047663420439,0.0025934537407010794,0.0021673892624676228,0.9885349869728088,0.0026461188681423664]},{"question_id":"auto_renews","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":7.345527410507202e-05,"mean_abs_dp":7.344409823417664e-05,"max_abs_dlogit":0.004774332046508789,"oracle_top2_margin":0.8997148089110851,"near_tie":false,"probs":[0.949783980846405,0.050216060131788254]},{"question_id":"liability_cap","type":"choice","n_options":4,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":9.322166442871094e-05,"mean_abs_dp":4.6607921831309795e-05,"max_abs_dlogit":0.006997942924499512,"oracle_top2_margin":0.9729512641206384,"near_tie":false,"probs":[0.9805073142051697,0.006575106643140316,0.0052400073036551476,0.007677624002099037]},{"question_id":"assign_without_consent","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":5.674734711647034e-05,"mean_abs_dp":5.6745484471321106e-05,"max_abs_dlogit":0.004107117652893066,"oracle_top2_margin":0.9651009291410446,"near_tie":false,"probs":[0.01750626042485237,0.9824936985969543]},{"question_id":"client_risk","type":"score","n_options":5,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00045412778854370117,"mean_abs_dp":0.00018166378140449524,"max_abs_dlogit":0.004863634705543518,"oracle_top2_margin":0.11966484785079956,"near_tie":false,"probs":[0.03787040710449219,0.09017660468816757,0.46629267930984497,0.34713777899742126,0.05852257087826729]},{"question_id":"uptime_target","type":"choice","n_options":4,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":7.593631744384766e-05,"mean_abs_dp":3.793195355683565e-05,"max_abs_dlogit":0.010350704193115234,"oracle_top2_margin":0.9738647546619177,"near_tie":false,"probs":[0.007181195542216301,0.9809327125549316,0.0061094933189451694,0.005776502192020416]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00045412778854370117,"mean_abs_dp":7.024284621531314e-05,"max_abs_dlogit":0.010350704193115234},{"id":"own_j01","arm":"text","variant":"base","tokens":476,"calls":4,"padded_tokens":512,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.9825300829834305,"wall_seconds":0.9932772910105996,"finite":true,"index":7,"shard":"shard_00","first_in_process":false,"source":"own_json","hidden":{"max_abs_diff":1.4357128143310547,"ref_absmax":49.30904769897461,"rel_max_abs_diff":0.029116620201142328,"min_pos_cos":0.9990809305507737,"min_pos_cos_index":179,"mean_pos_cos":0.9999750513004001,"positions":476,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"status","type":"choice","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0005138814449310303,"mean_abs_dp":0.00025694165378808975,"max_abs_dlogit":0.015454769134521484,"oracle_top2_margin":0.83003219217062,"near_tie":false,"probs":[0.0120798135176301,0.07330567389726639,0.9023838639259338,0.012230648659169674]},{"question_id":"large","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":1.3243407011032104e-06,"mean_abs_dp":1.317821443080902e-06,"max_abs_dlogit":0.005562722682952881,"oracle_top2_margin":0.9913223842158914,"near_tie":false,"probs":[0.004337490536272526,0.99566251039505]},{"question_id":"priority","type":"score","n_options":4,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0014041215181350708,"mean_abs_dp":0.0007324721664190292,"max_abs_dlogit":0.016757726669311523,"oracle_top2_margin":0.19878698885440826,"near_tie":false,"probs":[0.2478451281785965,0.44748637080192566,0.23697495460510254,0.06769351661205292]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0014041215181350708,"mean_abs_dp":0.0003960290923714638,"max_abs_dlogit":0.016757726669311523},{"id":"own_j02","arm":"text","variant":"base","tokens":475,"calls":4,"padded_tokens":512,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.9838907089724671,"wall_seconds":0.9945375410025008,"finite":true,"index":8,"shard":"shard_00","first_in_process":false,"source":"own_json","hidden":{"max_abs_diff":1.4132022857666016,"ref_absmax":48.36116027832031,"rel_max_abs_diff":0.029221844092109636,"min_pos_cos":0.9975227952578837,"min_pos_cos_index":352,"mean_pos_cos":0.9999645436377326,"positions":475,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"can_ship","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":2.7239322662353516e-05,"mean_abs_dp":2.7234666049480438e-05,"max_abs_dlogit":0.0029296875,"oracle_top2_margin":0.9560996349900961,"near_tie":false,"probs":[0.021977409720420837,0.978022575378418]},{"question_id":"blocker","type":"choice","n_options":4,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":7.909536361694336e-05,"mean_abs_dp":4.1366321966052055e-05,"max_abs_dlogit":0.0056765079498291016,"oracle_top2_margin":0.9330523833632469,"near_tie":false,"probs":[0.9525965452194214,0.017821501940488815,0.019644679501652718,0.009937293827533722]},{"question_id":"gift","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":4.303455352783203e-05,"mean_abs_dp":4.299357533454895e-05,"max_abs_dlogit":0.0066950321197509766,"oracle_top2_margin":0.9137708581984043,"near_tie":false,"probs":[0.9568424224853516,0.04315755143761635]},{"question_id":"priority","type":"score","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0013540089130401611,"mean_abs_dp":0.0009026875098546346,"max_abs_dlogit":0.00988537073135376,"oracle_top2_margin":0.18996360898017883,"near_tie":false,"probs":[0.2830340564250946,0.4749869704246521,0.24197900295257568]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0013540089130401611,"mean_abs_dp":0.00027399857274510646,"max_abs_dlogit":0.00988537073135376},{"id":"own_j03","arm":"text","variant":"base","tokens":585,"calls":5,"padded_tokens":640,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":1.2292022090114187,"wall_seconds":1.2401640829921234,"finite":true,"index":9,"shard":"shard_00","first_in_process":false,"source":"own_json","hidden":{"max_abs_diff":2.4131832122802734,"ref_absmax":51.966583251953125,"rel_max_abs_diff":0.04643721140141681,"min_pos_cos":0.9940176019825165,"min_pos_cos_index":318,"mean_pos_cos":0.9999342364602092,"positions":585,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"over_limit","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":2.5931745767593384e-05,"mean_abs_dp":2.5900080800056458e-05,"max_abs_dlogit":0.0012689828872680664,"oracle_top2_margin":0.9472900442779064,"near_tie":false,"probs":[0.9736708998680115,0.026329055428504944]},{"question_id":"first_violation","type":"choice","n_options":5,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00034862756729125977,"mean_abs_dp":0.00013943798840045928,"max_abs_dlogit":0.010752201080322266,"oracle_top2_margin":0.9380843341350555,"near_tie":false,"probs":[0.009668746031820774,0.008737987838685513,0.9558873176574707,0.017338132485747337,0.008367872796952724]},{"question_id":"trend","type":"choice","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":3.737211227416992e-05,"mean_abs_dp":2.4917845924695332e-05,"max_abs_dlogit":0.0040454864501953125,"oracle_top2_margin":0.9589130245149136,"near_tie":false,"probs":[0.012355031445622444,0.014398139901459217,0.9732468128204346]},{"question_id":"action","type":"score","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.001764833927154541,"mean_abs_dp":0.0008823997341096401,"max_abs_dlogit":0.008150219917297363,"oracle_top2_margin":0.053304851055145264,"near_tie":false,"probs":[0.016079841181635857,0.029870709404349327,0.5020299553871155,0.45201951265335083]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.001764833927154541,"mean_abs_dp":0.000310953041272504,"max_abs_dlogit":0.010752201080322266},{"id":"own_j04","arm":"text","variant":"base","tokens":569,"calls":5,"padded_tokens":640,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":1.2306923329888377,"wall_seconds":1.2415627920127008,"finite":true,"index":10,"shard":"shard_00","first_in_process":false,"source":"own_json","hidden":{"max_abs_diff":2.564862087368965,"ref_absmax":48.31694412231445,"rel_max_abs_diff":0.05308411229145641,"min_pos_cos":0.9927338004760681,"min_pos_cos_index":426,"mean_pos_cos":0.9999558565775565,"positions":569,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"duration","type":"choice","n_options":4,"argmax":3,"argmax_oracle":3,"argmax_equal":true,"max_abs_dp":0.007267773151397705,"mean_abs_dp":0.0036502135917544365,"max_abs_dlogit":0.02937138080596924,"oracle_top2_margin":0.10314184427261353,"near_tie":false,"probs":[0.04948344826698303,0.02478748932480812,0.4044654071331024,0.5212637186050415]},{"question_id":"recurring","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":2.1750107407569885e-05,"mean_abs_dp":2.169329673051834e-05,"max_abs_dlogit":0.003905773162841797,"oracle_top2_margin":0.9881622050888836,"near_tie":false,"probs":[0.005940624978393316,0.9940594434738159]},{"question_id":"accepted","type":"choice","n_options":5,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0003693103790283203,"mean_abs_dp":0.00016031619161367417,"max_abs_dlogit":0.008181571960449219,"oracle_top2_margin":0.8471281565725803,"near_tie":false,"probs":[0.040071871131658554,0.8916082382202148,0.04394925385713577,0.013648292981088161,0.0107222655788064]},{"question_id":"attendance_risk","type":"score","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0003015585243701935,"mean_abs_dp":0.00020104274153709412,"max_abs_dlogit":0.008128643035888672,"oracle_top2_margin":0.3122881054878235,"near_tie":false,"probs":[0.058748386800289154,0.626879870891571,0.3143717348575592]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.007267773151397705,"mean_abs_dp":0.001146353581654174,"max_abs_dlogit":0.02937138080596924},{"id":"img_01","arm":"g448","variant":"base","tokens":562,"calls":5,"padded_tokens":640,"grid":[14,14],"n_image_rows":196,"start":233,"amount":182,"image_tokens":196,"decode_seconds":1.228253000008408,"wall_seconds":1.247849417006364,"finite":true,"index":11,"shard":"shard_00","first_in_process":false,"source":"own_image","hidden":{"max_abs_diff":1.3809709548950195,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.028581556380898935,"min_pos_cos":0.9989077919222186,"min_pos_cos_index":369,"mean_pos_cos":0.9999779603519606,"positions":562,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"circles","type":"choice","n_options":6,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0006059408187866211,"mean_abs_dp":0.00020197235668698946,"max_abs_dlogit":0.01517033576965332,"oracle_top2_margin":0.9202973758801818,"near_tie":false,"probs":[0.009477371349930763,0.013735189102590084,0.9357385635375977,0.012536595575511456,0.014639263041317463,0.013873054645955563]},{"question_id":"more_circles","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.0007541775703430176,"mean_abs_dp":0.0007541552186012268,"max_abs_dlogit":0.009769022464752197,"oracle_top2_margin":0.5916850119829178,"near_tie":false,"probs":[0.7965966463088989,0.2034033238887787]},{"question_id":"clutter","type":"score","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.0005609095096588135,"mean_abs_dp":0.0003739502280950546,"max_abs_dlogit":0.010158106684684753,"oracle_top2_margin":0.28941109776496887,"near_tie":false,"probs":[0.34031200408935547,0.6286085247993469,0.03107944317162037]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0007541775703430176,"mean_abs_dp":0.00034927229650995946,"max_abs_dlogit":0.01517033576965332},{"id":"img_02","arm":"g448","variant":"base","tokens":598,"calls":5,"padded_tokens":640,"grid":[14,14],"n_image_rows":196,"start":233,"amount":182,"image_tokens":196,"decode_seconds":1.2285279170027934,"wall_seconds":1.2477862090163399,"finite":true,"index":12,"shard":"shard_00","first_in_process":false,"source":"own_image","hidden":{"max_abs_diff":0.8890724182128906,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.01840087465835632,"min_pos_cos":0.9984878666669235,"min_pos_cos_index":116,"mean_pos_cos":0.9999735182529346,"positions":598,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"tallest","type":"choice","n_options":5,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.00011307001113891602,"mean_abs_dp":4.5205093920230864e-05,"max_abs_dlogit":0.0032930970191955566,"oracle_top2_margin":0.9293951261788607,"near_tie":false,"probs":[0.011657328344881535,0.9480568766593933,0.0184891689568758,0.010638353414833546,0.011158312670886517]},{"question_id":"green_over_orange","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":6.330013275146484e-05,"mean_abs_dp":6.32554292678833e-05,"max_abs_dlogit":0.0033339262008666992,"oracle_top2_margin":0.8826808221638203,"near_tie":false,"probs":[0.9414036870002747,0.058596353977918625]},{"question_id":"spread","type":"score","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.000751987099647522,"mean_abs_dp":0.0003999723121523857,"max_abs_dlogit":0.013952255249023438,"oracle_top2_margin":0.3051571547985077,"near_tie":false,"probs":[0.04210535064339638,0.21668946743011475,0.5228291749954224,0.218376025557518]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.000751987099647522,"mean_abs_dp":0.00017749323424967852,"max_abs_dlogit":0.013952255249023438},{"id":"img_03","arm":"g448","variant":"base","tokens":627,"calls":5,"padded_tokens":640,"grid":[14,14],"n_image_rows":196,"start":233,"amount":182,"image_tokens":196,"decode_seconds":1.2272143329901155,"wall_seconds":1.2461730420181993,"finite":true,"index":13,"shard":"shard_00","first_in_process":false,"source":"own_image","hidden":{"max_abs_diff":3.210784912109375,"ref_absmax":48.316856384277344,"rel_max_abs_diff":0.06645268654427998,"min_pos_cos":0.9949953206842478,"min_pos_cos_index":79,"mean_pos_cos":0.9999530860260619,"positions":627,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"tens_digit","type":"choice","n_options":10,"argmax":4,"argmax_oracle":4,"argmax_equal":true,"max_abs_dp":0.0001302957534790039,"mean_abs_dp":2.605835907161236e-05,"max_abs_dlogit":0.004334092140197754,"oracle_top2_margin":0.9521092572249472,"near_tie":false,"probs":[0.004507158417254686,0.005108753219246864,0.004429214634001255,0.004810281563550234,0.9580347537994385,0.005786837078630924,0.004263828042894602,0.004629268776625395,0.004237048793584108,0.0041928160935640335]},{"question_id":"even","type":"noul","n_options":2,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.00045758485794067383,"mean_abs_dp":0.00045758113265037537,"max_abs_dlogit":0.011697888374328613,"oracle_top2_margin":0.9024685919284821,"near_tie":false,"probs":[0.0483081117272377,0.9516918659210205]},{"question_id":"readability","type":"score","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0006722211837768555,"mean_abs_dp":0.0004481549064318339,"max_abs_dlogit":0.011765003204345703,"oracle_top2_margin":0.7032996490597725,"near_tie":false,"probs":[0.06162792816758156,0.11702148616313934,0.8213505744934082]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0006722211837768555,"mean_abs_dp":0.0001680140383541584,"max_abs_dlogit":0.011765003204345703},{"id":"img_04","arm":"g448","variant":"base","tokens":690,"calls":6,"padded_tokens":768,"grid":[14,14],"n_image_rows":196,"start":233,"amount":182,"image_tokens":196,"decode_seconds":1.4744779169850517,"wall_seconds":1.4943158750247676,"finite":true,"index":14,"shard":"shard_00","first_in_process":false,"source":"own_image","hidden":{"max_abs_diff":2.84614877961576,"ref_absmax":48.31694793701172,"rel_max_abs_diff":0.05890580637100951,"min_pos_cos":0.98531842070341,"min_pos_cos_index":208,"mean_pos_cos":0.9998758803642411,"positions":690,"positions_below":{"0.99":2,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":2},"questions":[{"question_id":"total","type":"choice","n_options":4,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":7.724761962890625e-05,"mean_abs_dp":3.862055018544197e-05,"max_abs_dlogit":0.0021427273750305176,"oracle_top2_margin":0.9414124302566051,"near_tie":false,"probs":[0.016981467604637146,0.013094048015773296,0.9582615494728088,0.011662936769425869]},{"question_id":"paid_cash","type":"noul","n_options":2,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":0.0001327916979789734,"mean_abs_dp":0.00013273581862449646,"max_abs_dlogit":0.009533405303955078,"oracle_top2_margin":0.9361573942005634,"near_tie":false,"probs":[0.9679459929466248,0.03205407038331032]},{"question_id":"items","type":"choice","n_options":6,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00011730194091796875,"mean_abs_dp":3.908838455875715e-05,"max_abs_dlogit":0.004835784435272217,"oracle_top2_margin":0.9564370606094599,"near_tie":false,"probs":[0.005973236169666052,0.008283278904855251,0.9653788208961487,0.008801599964499474,0.005905931815505028,0.005657139699906111]},{"question_id":"legibility","type":"score","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.0004811286926269531,"mean_abs_dp":0.0003207574288050334,"max_abs_dlogit":0.003087639808654785,"oracle_top2_margin":0.7817753255367279,"near_tie":false,"probs":[0.048562511801719666,0.08441489189863205,0.8670225739479065]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.0004811286926269531,"mean_abs_dp":0.00010778376211722692,"max_abs_dlogit":0.009533405303955078},{"id":"semif_a3f18f3a63d45345942b","arm":"text","variant":"base","tokens":185,"calls":2,"padded_tokens":256,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.49118766697938554,"wall_seconds":0.5024167080118787,"finite":true,"index":15,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":1.2367324829101562,"ref_absmax":48.31694793701172,"rel_max_abs_diff":0.025596245949194053,"min_pos_cos":0.9985240924527983,"min_pos_cos_index":137,"mean_pos_cos":0.9999580583047597,"positions":185,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":0,"argmax_oracle":0,"argmax_equal":true,"max_abs_dp":8.922815322875977e-05,"mean_abs_dp":5.950282017389933e-05,"max_abs_dlogit":0.001846909523010254,"oracle_top2_margin":0.9059995450079441,"near_tie":false,"probs":[0.9378812909126282,0.030094658955931664,0.03202404826879501]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":8.922815322875977e-05,"mean_abs_dp":5.950282017389933e-05,"max_abs_dlogit":0.001846909523010254},{"id":"semif_f40beba9088c8db8bbd6","arm":"text","variant":"base","tokens":183,"calls":2,"padded_tokens":256,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.49076054198667407,"wall_seconds":0.5012072089884896,"finite":true,"index":16,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":0.6862659454345703,"ref_absmax":48.316871643066406,"rel_max_abs_diff":0.014203443271415756,"min_pos_cos":0.9995604733118414,"min_pos_cos_index":101,"mean_pos_cos":0.9999630899377169,"positions":183,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00010710954666137695,"mean_abs_dp":7.140015562375386e-05,"max_abs_dlogit":0.002959728240966797,"oracle_top2_margin":0.8385190591216087,"near_tie":false,"probs":[0.034286241978406906,0.06365527957677841,0.902058482170105]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00010710954666137695,"mean_abs_dp":7.140015562375386e-05,"max_abs_dlogit":0.002959728240966797},{"id":"semif_0b43ea8e24e74d621f1b","arm":"text","variant":"base","tokens":177,"calls":2,"padded_tokens":256,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.49146820898749866,"wall_seconds":0.5025109999987762,"finite":true,"index":17,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":0.661707878112793,"ref_absmax":48.31693649291992,"rel_max_abs_diff":0.013695153835130994,"min_pos_cos":0.9994656513988227,"min_pos_cos_index":115,"mean_pos_cos":0.9999779205028283,"positions":177,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":0.00017124414443969727,"mean_abs_dp":0.00011414537827173869,"max_abs_dlogit":0.002392411231994629,"oracle_top2_margin":0.9089595079421997,"near_tie":false,"probs":[0.02595246583223343,0.03239911049604416,0.9416484236717224]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00017124414443969727,"mean_abs_dp":0.00011414537827173869,"max_abs_dlogit":0.002392411231994629},{"id":"semif_942ec65dac0bfb9874f9","arm":"text","variant":"base","tokens":177,"calls":2,"padded_tokens":256,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.4909775829873979,"wall_seconds":0.5020164170127828,"finite":true,"index":18,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":0.9161415100097656,"ref_absmax":48.31693649291992,"rel_max_abs_diff":0.01896108438381667,"min_pos_cos":0.9994863188379982,"min_pos_cos_index":139,"mean_pos_cos":0.99996641027423,"positions":177,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":1,"argmax_oracle":1,"argmax_equal":true,"max_abs_dp":0.00024896860122680664,"mean_abs_dp":0.00016595485309759775,"max_abs_dlogit":0.003228425979614258,"oracle_top2_margin":0.7890182957053185,"near_tie":false,"probs":[0.09129466116428375,0.8798273205757141,0.02887800894677639]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":0.00024896860122680664,"mean_abs_dp":0.00016595485309759775,"max_abs_dlogit":0.003228425979614258},{"id":"semif_e0c140e9222d1bce5647","arm":"text","variant":"base","tokens":189,"calls":2,"padded_tokens":256,"grid":null,"n_image_rows":0,"start":1073741824,"amount":0,"image_tokens":0,"decode_seconds":0.4903078750066925,"wall_seconds":0.5009427079930902,"finite":true,"index":19,"shard":"shard_00","first_in_process":false,"source":"semif_authored144","hidden":{"max_abs_diff":0.7769775390625,"ref_absmax":48.31690979003906,"rel_max_abs_diff":0.016080861595637073,"min_pos_cos":0.9987808431479216,"min_pos_cos_index":66,"mean_pos_cos":0.9999658921190477,"positions":189,"positions_below":{"0.99":0,"0.9":0,"0.5":0},"positions_below_0.99_image_rows":0},"questions":[{"question_id":"decision","type":"choice","n_options":3,"argmax":2,"argmax_oracle":2,"argmax_equal":true,"max_abs_dp":2.7727335691452026e-05,"mean_abs_dp":1.849234104156494e-05,"max_abs_dlogit":0.0020508766174316406,"oracle_top2_margin":0.8911582641303539,"near_tie":false,"probs":[0.03843174874782562,0.031995534896850586,0.9295727014541626]}],"argmax_all_equal":true,"argmax_equal_non_near_tie":true,"max_abs_dp":2.7727335691452026e-05,"mean_abs_dp":1.849234104156494e-05,"max_abs_dlogit":0.0020508766174316406}],"compare_with":{"other":"<lane>/results/readout_fp16_pf64_full.json","other_chunk":64,"other_bundle":"clef_flash_decode_fp16_pf64","runs":20,"max_abs_dp":0.001710355281829834,"argmax_equal_runs":20,"hidden_max_abs_diff":0.734375,"vs_oracle_on_shared_runs":{"order":["other","this"],"max_abs_dp":[0.008227109909057617,0.007786452770233154],"min_pos_cos":[0.923740265229428,0.9479832238632915],"positions_below_0.99":[5,6]},"timing_warm_common_runs":{"other":{"S":64,"runs":19,"ms_per_call_median":134.4263959908858,"ms_per_token_median":2.188072858955526,"run_decode_seconds_median":1.2100042910024058,"calls_total":204},"this":{"S":128,"runs":19,"ms_per_call_median":246.25054199714214,"ms_per_token_median":2.13692451736964,"run_decode_seconds_median":1.2272143329901155,"calls_total":107}}},"note":"round 4 step 5: S=128 trial on 20 runs (own_text 5, long 2, json 4, own_image g448 4, SemIf 5), times contended","generated_at":"2026-10-03T04:20:55+00:00"}},"grid_price":{"schema":"clef-flash-grid-price/1","what":"native (processor grid) vs g256 / g448 / g672 / g896 (PIL BICUBIC to the square tile first) on every image record of the fixture, CPU fp32 oracle (the checkpoint's joint_schema_model.py); dp = per-option probability difference","oracle":"<lane>/oracle/records_oracle.json","oracle_sha256":"d8adaa746a5d575e654212ca78f5634af334aaa62eb53ef150d5d5925b1fa7c0","oracle_large":"<lane>/oracle/records_oracle_large.json","oracle_large_sha256":"93e0c7115ced533a01a5f7ee5b10fa6267d1dc5d28ca75cf0b3cb2724f58ab42","image_records":14,"questions":50,"arms":{"native":{"questions":50,"gold_questions":38,"gold_correct":35,"mean_image_tokens":412.0,"mean_tokens":830.3571428571429},"g256":{"questions":50,"argmax_agree_with_native":46,"argmax_agree_rate":0.92,"flips":["img_07/odd_direction","img_07/odd_position","img_08/rooms","photo_01/kinds"],"max_abs_dp_vs_native":0.9586424874141812,"mean_of_question_max_abs_dp":0.06773983755148948,"mean_abs_dp_vs_native":0.039413047045624504,"gold_questions":38,"gold_correct":33,"mean_image_tokens":64.0,"mean_tokens":482.35714285714283},"g448":{"questions":50,"argmax_agree_with_native":48,"argmax_agree_rate":0.96,"flips":["img_01/clutter","img_08/rooms"],"max_abs_dp_vs_native":0.4363647624850273,"mean_of_question_max_abs_dp":0.03127590139396488,"mean_abs_dp_vs_native":0.01835391184478067,"gold_questions":38,"gold_correct":36,"mean_image_tokens":196.0,"mean_tokens":614.3571428571429},"g672":{"questions":50,"argmax_agree_with_native":48,"argmax_agree_rate":0.96,"flips":["img_01/clutter","img_08/rooms"],"max_abs_dp_vs_native":0.3499705493450165,"mean_of_question_max_abs_dp":0.026955362744629383,"mean_abs_dp_vs_native":0.016234463536801438,"gold_questions":38,"gold_correct":36,"mean_image_tokens":441.0,"mean_tokens":859.3571428571429},"g896":{"questions":50,"argmax_agree_with_native":48,"argmax_agree_rate":0.96,"flips":["img_01/clutter","photo_01/limes"],"max_abs_dp_vs_native":0.24421268701553345,"mean_of_question_max_abs_dp":0.02194898693356663,"mean_abs_dp_vs_native":0.012696248807168253,"gold_questions":38,"gold_correct":36,"mean_image_tokens":784.0,"mean_tokens":1202.357142857143}},"records":[{"id":"img_01","image_files":["shapes_224.png"],"image_size_in":[224,224],"native_grid_thw":[1,16,16],"arms":{"native":{"tokens":430,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":13.29156366699317},"g256":{"tokens":430,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":11.859875458001625},"g448":{"tokens":562,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":14.223177167004906},"g672":{"tokens":807,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":19.60751245901338},"g896":{"tokens":1150,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":22.80378675000975}}},{"id":"img_02","image_files":["bars_640x480.png"],"image_size_in":[640,480],"native_grid_thw":[1,30,40],"arms":{"native":{"tokens":702,"n_image_tokens":300,"grid_thw":[1,30,40],"wall_s":13.944828249994316},"g256":{"tokens":466,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":13.352636583993444},"g448":{"tokens":598,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":13.760779459000332},"g672":{"tokens":843,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":19.09973454201827},"g896":{"tokens":1186,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":23.199003957997775}}},{"id":"img_03","image_files":["sevenseg_47_300x200.png"],"image_size_in":[300,200],"native_grid_thw":[1,14,20],"arms":{"native":{"tokens":501,"n_image_tokens":70,"grid_thw":[1,14,20],"wall_s":13.483955624993541},"g256":{"tokens":495,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":12.05536287499126},"g448":{"tokens":627,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":14.356350165995536},"g672":{"tokens":872,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":19.63495958299609},"g896":{"tokens":1215,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":23.300504540995462}}},{"id":"img_04","image_files":["receipt_1024x768.png"],"image_size_in":[1024,768],"native_grid_thw":[1,48,64],"arms":{"native":{"tokens":1262,"n_image_tokens":768,"grid_thw":[1,48,64],"wall_s":18.227609458001098},"g256":{"tokens":558,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":13.97465312499844},"g448":{"tokens":690,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":14.203698957993765},"g672":{"tokens":935,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":20.09835270800977},"g896":{"tokens":1278,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":23.980400875007035}}},{"id":"img_05","image_files":["signal_green_256x240.png"],"image_size_in":[256,240],"native_grid_thw":[1,16,16],"arms":{"native":{"tokens":464,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":13.39318179199472},"g256":{"tokens":464,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":11.322417249990394},"g448":{"tokens":596,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":14.262977042002603},"g672":{"tokens":841,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":19.23166045799735},"g896":{"tokens":1184,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":23.619578000012552}}},{"id":"img_06","image_files":["gauge_300x200.png"],"image_size_in":[300,200],"native_grid_thw":[1,14,20],"arms":{"native":{"tokens":471,"n_image_tokens":70,"grid_thw":[1,14,20],"wall_s":11.57589658300276},"g256":{"tokens":465,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":14.050204874991323},"g448":{"tokens":597,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":13.484710707998602},"g672":{"tokens":842,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":16.91286520799622},"g896":{"tokens":1185,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":20.050392834004015}}},{"id":"img_07","image_files":["arrows_640x480.png"],"image_size_in":[640,480],"native_grid_thw":[1,30,40],"arms":{"native":{"tokens":662,"n_image_tokens":300,"grid_thw":[1,30,40],"wall_s":13.47323391700047},"g256":{"tokens":426,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":11.398928082999191},"g448":{"tokens":558,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":14.325285208004061},"g672":{"tokens":803,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":16.63315562499338},"g896":{"tokens":1146,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":19.91996233401005}}},{"id":"img_08","image_files":["floorplan_1024x768.png"],"image_size_in":[1024,768],"native_grid_thw":[1,48,64],"arms":{"native":{"tokens":1226,"n_image_tokens":768,"grid_thw":[1,48,64],"wall_s":18.114923832996283},"g256":{"tokens":522,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":12.582467332991655},"g448":{"tokens":654,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":14.009094374996494},"g672":{"tokens":899,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":17.16781445799279},"g896":{"tokens":1242,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":20.45890170900384}}},{"id":"img_09","image_files":["circles_640x480.png"],"image_size_in":[640,480],"native_grid_thw":[1,30,40],"arms":{"native":{"tokens":696,"n_image_tokens":300,"grid_thw":[1,30,40],"wall_s":14.006880375003675},"g256":{"tokens":460,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":12.552474583004368},"g448":{"tokens":592,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":15.057918749997043},"g672":{"tokens":837,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":15.900240832997952},"g896":{"tokens":1180,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":17.832876208005473}}},{"id":"img_10","image_files":["clock_3oclock_256x240.png"],"image_size_in":[256,240],"native_grid_thw":[1,16,16],"arms":{"native":{"tokens":405,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":13.08876508299727},"g256":{"tokens":405,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":11.427527833991917},"g448":{"tokens":537,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":14.099708250010735},"g672":{"tokens":782,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":14.980308375001187},"g896":{"tokens":1125,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":17.524573542003054}}},{"id":"photo_01","image_files":["photo_fruit.png"],"image_size_in":[960,1280],"native_grid_thw":[1,80,60],"arms":{"native":{"tokens":1692,"n_image_tokens":1200,"grid_thw":[1,80,60],"wall_s":21.603107415998238},"g256":{"tokens":556,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":14.455852874991251},"g448":{"tokens":688,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":13.866414333999273},"g672":{"tokens":933,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":15.741525332996389},"g896":{"tokens":1276,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":18.58251200002269}}},{"id":"photo_02","image_files":["photo_cat.png"],"image_size_in":[960,640],"native_grid_thw":[1,40,60],"arms":{"native":{"tokens":1035,"n_image_tokens":600,"grid_thw":[1,40,60],"wall_s":16.609171208998305},"g256":{"tokens":499,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":14.068744042000617},"g448":{"tokens":631,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":14.088534084003186},"g672":{"tokens":876,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":15.408225291001145},"g896":{"tokens":1219,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":17.984997292020125}}},{"id":"photo_03","image_files":["photo_lake.png"],"image_size_in":[960,640],"native_grid_thw":[1,40,60],"arms":{"native":{"tokens":1041,"n_image_tokens":600,"grid_thw":[1,40,60],"wall_s":16.684209416998783},"g256":{"tokens":505,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":11.751078083005268},"g448":{"tokens":637,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":13.939065459009726},"g672":{"tokens":882,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":15.517343041981803},"g896":{"tokens":1225,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":18.12444799998775}}},{"id":"photo_04","image_files":["photo_coffee.png"],"image_size_in":[960,635],"native_grid_thw":[1,40,60],"arms":{"native":{"tokens":1038,"n_image_tokens":600,"grid_thw":[1,40,60],"wall_s":16.653283374995226},"g256":{"tokens":502,"n_image_tokens":64,"grid_thw":[1,16,16],"wall_s":12.012700666004093},"g448":{"tokens":634,"n_image_tokens":196,"grid_thw":[1,28,28],"wall_s":14.184826875003637},"g672":{"tokens":879,"n_image_tokens":441,"grid_thw":[1,42,42],"wall_s":15.498084791994188},"g896":{"tokens":1222,"n_image_tokens":784,"grid_thw":[1,56,56],"wall_s":18.46445670799585}}}],"per_question":[{"id":"img_01","question_id":"circles","type":"choice","n_options":6,"native_argmax":"3","native_top2_margin":0.9245060188695788,"gold":"3","g256":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.0,"mean_abs_dp":0.0,"top2_margin":0.9245060188695788},"g448":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.0033605098724365234,"mean_abs_dp":0.0011201708888014157,"top2_margin":0.9202973758801818},"g672":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.0004398757591843605,"mean_abs_dp":0.00027248403057456017,"top2_margin":0.9243332957848907},"g896":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.004841744899749756,"mean_abs_dp":0.0016139067399005096,"top2_margin":0.9302583392709494}},{"id":"img_01","question_id":"more_circles","type":"noul","n_options":2,"native_argmax":"true","native_top2_margin":0.6116609573364258,"gold":"true","g256":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.0,"mean_abs_dp":0.0,"top2_margin":0.6116609573364258},"g448":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.009988009929656982,"mean_abs_dp":0.009987972676753998,"top2_margin":0.5916850119829178},"g672":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.02029895782470703,"mean_abs_dp":0.02029893547296524,"top2_margin":0.5710630863904953},"g896":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.00524100661277771,"mean_abs_dp":0.005240991711616516,"top2_margin":0.6221429407596588}},{"id":"img_01","question_id":"clutter","type":"score","n_options":3,"native_argmax":"0","native_top2_margin":0.002184569835662842,"gold":null,"g256":{"argmax":"0","argmax_agree":true,"max_abs_dp":0.0,"mean_abs_dp":0.0,"top2_margin":0.002184569835662842},"g448":{"argmax":"1","argmax_agree":false,"max_abs_dp":0.15081703662872314,"mean_abs_dp":0.10054466625054677,"top2_margin":0.28941109776496887},"g672":{"argmax":"1","argmax_agree":false,"max_abs_dp":0.1444716453552246,"mean_abs_dp":0.09631439050038655,"top2_margin":0.27659478783607483},"g896":{"argmax":"1","argmax_agree":false,"max_abs_dp":0.1691652536392212,"mean_abs_dp":0.11277680844068527,"top2_margin":0.3222171366214752}},{"id":"img_02","question_id":"tallest","type":"choice","n_options":5,"native_argmax":"green","native_top2_margin":0.9413488861173391,"gold":"green","g256":{"argmax":"green","argmax_agree":true,"max_abs_dp":0.001444399356842041,"mean_abs_dp":0.0008871341124176979,"top2_margin":0.9406779743731022},"g448":{"argmax":"green","argmax_agree":true,"max_abs_dp":0.0074631571769714355,"mean_abs_dp":0.002985232695937157,"top2_margin":0.9293951261788607},"g672":{"argmax":"green","argmax_agree":true,"max_abs_dp":0.0012871380895376205,"mean_abs_dp":0.000514877773821354,"top2_margin":0.9396401643753052},"g896":{"argmax":"green","argmax_agree":true,"max_abs_dp":0.004544377326965332,"mean_abs_dp":0.0018177740275859833,"top2_margin":0.9484982695430517}},{"id":"img_02","question_id":"green_over_orange","type":"noul","n_options":2,"native_argmax":"true","native_top2_margin":0.8825415037572384,"gold":"true","g256":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.007141325622797012,"mean_abs_dp":0.007141308858990669,"top2_margin":0.8968241214752197},"g448":{"argmax":"true","argmax_agree":true,"max_abs_dp":6.970018148422241e-05,"mean_abs_dp":6.965920329093933e-05,"top2_margin":0.8826808221638203},"g672":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.00899958610534668,"mean_abs_dp":0.008999543264508247,"top2_margin":0.8645424172282219},"g896":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.006940845400094986,"mean_abs_dp":0.006940843537449837,"top2_margin":0.8964231908321381}},{"id":"img_02","question_id":"spread","type":"score","n_options":4,"native_argmax":"2","native_top2_margin":0.30996087193489075,"gold":null,"g256":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.012699536979198456,"mean_abs_dp":0.010209256783127785,"top2_margin":0.29390622675418854},"g448":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.0048482418060302734,"mean_abs_dp":0.0024463720619678497,"top2_margin":0.3051571547985077},"g672":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.04521700739860535,"mean_abs_dp":0.022608495317399502,"top2_margin":0.2644205093383789},"g896":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.05257529020309448,"mean_abs_dp":0.02658651489764452,"top2_margin":0.24792328476905823}},{"id":"img_03","question_id":"tens_digit","type":"choice","n_options":10,"native_argmax":"4","native_top2_margin":0.9546868926845491,"gold":"4","g256":{"argmax":"4","argmax_agree":true,"max_abs_dp":0.018707752227783203,"mean_abs_dp":0.003741557290777564,"top2_margin":0.9320426946505904},"g448":{"argmax":"4","argmax_agree":true,"max_abs_dp":0.0021187663078308105,"mean_abs_dp":0.0004237465094774961,"top2_margin":0.9521092572249472},"g672":{"argmax":"4","argmax_agree":true,"max_abs_dp":0.0002467036247253418,"mean_abs_dp":8.664303459227085e-05,"top2_margin":0.9550877632573247},"g896":{"argmax":"4","argmax_agree":true,"max_abs_dp":0.0007406473159790039,"mean_abs_dp":0.000177582330070436,"top2_margin":0.9556458569131792}},{"id":"img_03","question_id":"even","type":"noul","n_options":2,"native_argmax":"false","native_top2_margin":0.84402646869421,"gold":"false","g256":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.376726858317852,"mean_abs_dp":0.37672683224081993,"top2_margin":0.09057280421257019},"g448":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.0292210653424263,"mean_abs_dp":0.029221061617136,"top2_margin":0.9024685919284821},"g672":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.03253275156021118,"mean_abs_dp":0.03253272920846939,"top2_margin":0.9090919271111488},"g896":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.03396111726760864,"mean_abs_dp":0.03396109864115715,"top2_margin":0.9119486659765244}},{"id":"img_03","question_id":"readability","type":"score","n_options":3,"native_argmax":"2","native_top2_margin":0.6572192162275314,"gold":null,"g256":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.06527656316757202,"mean_abs_dp":0.04351769884427389,"top2_margin":0.540686309337616},"g448":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.02446460723876953,"mean_abs_dp":0.01630973940094312,"top2_margin":0.7032996490597725},"g672":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.05100369453430176,"mean_abs_dp":0.034002469231685005,"top2_margin":0.7462673112750053},"g896":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.05378270149230957,"mean_abs_dp":0.03585513556996981,"top2_margin":0.7515735179185867}},{"id":"img_04","question_id":"total","type":"choice","n_options":4,"native_argmax":"6.75","native_top2_margin":0.9573134584352374,"gold":"6.75","g256":{"argmax":"6.75","argmax_agree":true,"max_abs_dp":0.008811354637145996,"mean_abs_dp":0.004405679414048791,"top2_margin":0.9446959812194109},"g448":{"argmax":"6.75","argmax_agree":true,"max_abs_dp":0.011894047260284424,"mean_abs_dp":0.005947024328634143,"top2_margin":0.9414124302566051},"g672":{"argmax":"6.75","argmax_agree":true,"max_abs_dp":0.0031487056985497475,"mean_abs_dp":0.001595688285306096,"top2_margin":0.9515379164367914},"g896":{"argmax":"6.75","argmax_agree":true,"max_abs_dp":0.006317132152616978,"mean_abs_dp":0.0032093338668346405,"top2_margin":0.9452398288995028}},{"id":"img_04","question_id":"paid_cash","type":"noul","n_options":2,"native_argmax":"true","native_top2_margin":0.9134139567613602,"gold":"true","g256":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.0062201544642448425,"mean_abs_dp":0.006220128387212753,"top2_margin":0.9258542135357857},"g448":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.011371765285730362,"mean_abs_dp":0.011371718719601631,"top2_margin":0.9361573942005634},"g672":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.013385418802499771,"mean_abs_dp":0.013385416939854622,"top2_margin":0.8866431228816509},"g896":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.016424596309661865,"mean_abs_dp":0.01642456278204918,"top2_margin":0.8805648311972618}},{"id":"img_04","question_id":"items","type":"choice","n_options":6,"native_argmax":"3","native_top2_margin":0.9510575840249658,"gold":"3","g256":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.003346264362335205,"mean_abs_dp":0.0011154323971519868,"top2_margin":0.9453170308843255},"g448":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.004315853118896484,"mean_abs_dp":0.0014386198793848355,"top2_margin":0.9564370606094599},"g672":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.013542473316192627,"mean_abs_dp":0.004514172673225403,"top2_margin":0.935364255681634},"g896":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.019296646118164062,"mean_abs_dp":0.006432207766920328,"top2_margin":0.928331327624619}},{"id":"img_04","question_id":"legibility","type":"score","n_options":3,"native_argmax":"2","native_top2_margin":0.819927416741848,"gold":null,"g256":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.02724522352218628,"mean_abs_dp":0.018163484210769337,"top2_margin":0.8597390353679657},"g448":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.020652413368225098,"mean_abs_dp":0.013768274337053299,"top2_margin":0.7817753255367279},"g672":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.0414242148399353,"mean_abs_dp":0.027616151918967564,"top2_margin":0.7534179911017418},"g896":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.014018654823303223,"mean_abs_dp":0.009345797200997671,"top2_margin":0.8003473579883575}},{"id":"img_05","question_id":"action","type":"choice","n_options":3,"native_argmax":"go","native_top2_margin":0.944669371470809,"gold":"go","g256":{"argmax":"go","argmax_agree":true,"max_abs_dp":0.0,"mean_abs_dp":0.0,"top2_margin":0.944669371470809},"g448":{"argmax":"go","argmax_agree":true,"max_abs_dp":0.0019410252571105957,"mean_abs_dp":0.0012939988325039546,"top2_margin":0.9480947684496641},"g672":{"argmax":"go","argmax_agree":true,"max_abs_dp":0.002283155918121338,"mean_abs_dp":0.0015220828354358673,"top2_margin":0.9485678691416979},"g896":{"argmax":"go","argmax_agree":true,"max_abs_dp":0.0025562644004821777,"mean_abs_dp":0.0017041672642032306,"top2_margin":0.9490423649549484}},{"id":"img_05","question_id":"lit_color","type":"choice","n_options":3,"native_argmax":"green","native_top2_margin":0.9751403219997883,"gold":"green","g256":{"argmax":"green","argmax_agree":true,"max_abs_dp":0.0,"mean_abs_dp":0.0,"top2_margin":0.9751403219997883},"g448":{"argmax":"green","argmax_agree":true,"max_abs_dp":0.00024366378784179688,"mean_abs_dp":0.00016245897859334946,"top2_margin":0.9755466179922223},"g672":{"argmax":"green","argmax_agree":true,"max_abs_dp":0.0004839301109313965,"mean_abs_dp":0.0003226551537712415,"top2_margin":0.9758316380903125},"g896":{"argmax":"green","argmax_agree":true,"max_abs_dp":0.00127488374710083,"mean_abs_dp":0.0008499477989971638,"top2_margin":0.9770843242295086}},{"id":"img_05","question_id":"may_proceed","type":"noul","n_options":2,"native_argmax":"true","native_top2_margin":0.8753064572811127,"gold":"true","g256":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.0,"mean_abs_dp":0.0,"top2_margin":0.8753064572811127},"g448":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.0009605288505554199,"mean_abs_dp":0.0009605251252651215,"top2_margin":0.8733854070305824},"g672":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.003147996962070465,"mean_abs_dp":0.0031479783356189728,"top2_margin":0.8690105006098747},"g896":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.0033192038536071777,"mean_abs_dp":0.0033191628754138947,"top2_margin":0.8686681315302849}},{"id":"img_05","question_id":"caution","type":"score","n_options":3,"native_argmax":"0","native_top2_margin":0.7880274653434753,"gold":null,"g256":{"argmax":"0","argmax_agree":true,"max_abs_dp":0.0,"mean_abs_dp":0.0,"top2_margin":0.7880274653434753},"g448":{"argmax":"0","argmax_agree":true,"max_abs_dp":0.01894056797027588,"mean_abs_dp":0.012627036000291506,"top2_margin":0.8226336687803268},"g672":{"argmax":"0","argmax_agree":true,"max_abs_dp":0.023555219173431396,"mean_abs_dp":0.01570350117981434,"top2_margin":0.8307851105928421},"g896":{"argmax":"0","argmax_agree":true,"max_abs_dp":0.025873780250549316,"mean_abs_dp":0.01724916696548462,"top2_margin":0.834835447371006}},{"id":"img_06","question_id":"in_red_zone","type":"noul","n_options":2,"native_argmax":"true","native_top2_margin":0.29172810912132263,"gold":"true","g256":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.06051003932952881,"mean_abs_dp":0.060510024428367615,"top2_margin":0.1707080602645874},"g448":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.1371050477027893,"mean_abs_dp":0.13710501790046692,"top2_margin":0.017518073320388794},"g672":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.03584057092666626,"mean_abs_dp":0.03584057092666626,"top2_margin":0.2200469672679901},"g896":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.017017662525177002,"mean_abs_dp":0.017017662525177002,"top2_margin":0.32576343417167664}},{"id":"img_06","question_id":"reading","type":"choice","n_options":4,"native_argmax":"75_100","native_top2_margin":0.30915674567222595,"gold":"75_100","g256":{"argmax":"75_100","argmax_agree":true,"max_abs_dp":0.12595945596694946,"mean_abs_dp":0.06297974474728107,"top2_margin":0.06736165285110474},"g448":{"argmax":"75_100","argmax_agree":true,"max_abs_dp":0.13194724917411804,"mean_abs_dp":0.0659736453089863,"top2_margin":0.05165323615074158},"g672":{"argmax":"75_100","argmax_agree":true,"max_abs_dp":0.01672789454460144,"mean_abs_dp":0.00895561114884913,"top2_margin":0.2784789800643921},"g896":{"argmax":"75_100","argmax_agree":true,"max_abs_dp":0.023340344429016113,"mean_abs_dp":0.012904499424621463,"top2_margin":0.35490450263023376}},{"id":"img_06","question_id":"alarm","type":"score","n_options":4,"native_argmax":"2","native_top2_margin":0.39862820506095886,"gold":null,"g256":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.04605555534362793,"mean_abs_dp":0.023027796298265457,"top2_margin":0.32940465211868286},"g448":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.05107396841049194,"mean_abs_dp":0.02553701400756836,"top2_margin":0.3209385871887207},"g672":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.029607951641082764,"mean_abs_dp":0.01978938654065132,"top2_margin":0.3789909780025482},"g896":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.023124828934669495,"mean_abs_dp":0.0149086844176054,"top2_margin":0.41506050527095795}},{"id":"img_07","question_id":"odd_direction","type":"choice","n_options":4,"native_argmax":"up","native_top2_margin":0.9635526714846492,"gold":"up","g256":{"argmax":"left","argmax_agree":false,"max_abs_dp":0.9586424874141812,"mean_abs_dp":0.4793212389340624,"top2_margin":0.7762424126267433},"g448":{"argmax":"up","argmax_agree":true,"max_abs_dp":0.009475350379943848,"mean_abs_dp":0.004737678333185613,"top2_margin":0.9483929080888629},"g672":{"argmax":"up","argmax_agree":true,"max_abs_dp":0.004911482334136963,"mean_abs_dp":0.0024557658471167088,"top2_margin":0.9695106875151396},"g896":{"argmax":"up","argmax_agree":true,"max_abs_dp":0.007982432842254639,"mean_abs_dp":0.003991216653957963,"top2_margin":0.9741495540365577}},{"id":"img_07","question_id":"odd_position","type":"choice","n_options":4,"native_argmax":"2","native_top2_margin":0.8184187300503254,"gold":"2","g256":{"argmax":"1","argmax_agree":false,"max_abs_dp":0.5842194259166718,"mean_abs_dp":0.29210976138710976,"top2_margin":0.2659342586994171},"g448":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.02908390760421753,"mean_abs_dp":0.016630837693810463,"top2_margin":0.7735076621174812},"g672":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.010403215885162354,"mean_abs_dp":0.006332787685096264,"top2_margin":0.8296146914362907},"g896":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.0745089054107666,"mean_abs_dp":0.03725444385781884,"top2_margin":0.9186777081340551}},{"id":"img_07","question_id":"all_same","type":"noul","n_options":2,"native_argmax":"false","native_top2_margin":0.9710730528458953,"gold":"false","g256":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.11057867761701345,"mean_abs_dp":0.11057866690680385,"top2_margin":0.7499157190322876},"g448":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.008712922222912312,"mean_abs_dp":0.008712875191122293,"top2_margin":0.9536473024636507},"g672":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.000545799732208252,"mean_abs_dp":0.0005457829684019089,"top2_margin":0.9721646187826991},"g896":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.006715449038892984,"mean_abs_dp":0.006715432973578572,"top2_margin":0.9845039187930524}},{"id":"img_08","question_id":"largest_room","type":"choice","n_options":4,"native_argmax":"living","native_top2_margin":0.8977233991026878,"gold":"living","g256":{"argmax":"living","argmax_agree":true,"max_abs_dp":0.004423618316650391,"mean_abs_dp":0.002211815444752574,"top2_margin":0.8920186869800091},"g448":{"argmax":"living","argmax_agree":true,"max_abs_dp":0.021512866020202637,"mean_abs_dp":0.010756430448964238,"top2_margin":0.8663040027022362},"g672":{"argmax":"living","argmax_agree":true,"max_abs_dp":0.012105047702789307,"mean_abs_dp":0.006052530836313963,"top2_margin":0.8818609900772572},"g896":{"argmax":"living","argmax_agree":true,"max_abs_dp":0.010036170482635498,"mean_abs_dp":0.005018095253035426,"top2_margin":0.8830238506197929}},{"id":"img_08","question_id":"rooms","type":"choice","n_options":5,"native_argmax":"4","native_top2_margin":0.14287957549095154,"gold":"5","g256":{"argmax":"5","argmax_agree":false,"max_abs_dp":0.3635532855987549,"mean_abs_dp":0.14696370381861926,"top2_margin":0.5571556091308594},"g448":{"argmax":"5","argmax_agree":false,"max_abs_dp":0.4363647624850273,"mean_abs_dp":0.18151442669332027,"top2_margin":0.723192386329174},"g672":{"argmax":"5","argmax_agree":false,"max_abs_dp":0.3499705493450165,"mean_abs_dp":0.13998823557049037,"top2_margin":0.5307295769453049},"g896":{"argmax":"4","argmax_agree":true,"max_abs_dp":0.053112149238586426,"mean_abs_dp":0.021244850382208823,"top2_margin":0.19639578461647034}},{"id":"img_08","question_id":"bath_next_to_bed","type":"noul","n_options":2,"native_argmax":"false","native_top2_margin":0.384887158870697,"gold":"true","g256":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.004789203405380249,"mean_abs_dp":0.0047891587018966675,"top2_margin":0.39446547627449036},"g448":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.029989242553710938,"mean_abs_dp":0.02998921275138855,"top2_margin":0.4448655843734741},"g672":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.09985172748565674,"mean_abs_dp":0.09985166788101196,"top2_margin":0.584590494632721},"g896":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.019309520721435547,"mean_abs_dp":0.019309476017951965,"top2_margin":0.3462682068347931}},{"id":"img_08","question_id":"complexity","type":"score","n_options":3,"native_argmax":"1","native_top2_margin":0.4244216978549957,"gold":null,"g256":{"argmax":"1","argmax_agree":true,"max_abs_dp":0.043758511543273926,"mean_abs_dp":0.029172318677107494,"top2_margin":0.5075182020664215},"g448":{"argmax":"1","argmax_agree":true,"max_abs_dp":0.042599618434906006,"mean_abs_dp":0.028399755557378132,"top2_margin":0.5081702172756195},"g672":{"argmax":"1","argmax_agree":true,"max_abs_dp":0.030592799186706543,"mean_abs_dp":0.020395213117202122,"top2_margin":0.47784312069416046},"g896":{"argmax":"1","argmax_agree":true,"max_abs_dp":0.05377243459224701,"mean_abs_dp":0.035848286002874374,"top2_margin":0.5057156831026077}},{"id":"img_09","question_id":"largest","type":"choice","n_options":3,"native_argmax":"red","native_top2_margin":0.96602054964751,"gold":"red","g256":{"argmax":"red","argmax_agree":true,"max_abs_dp":0.0015787482261657715,"mean_abs_dp":0.0010525180647770564,"top2_margin":0.9686121270060539},"g448":{"argmax":"red","argmax_agree":true,"max_abs_dp":0.00048798322677612305,"mean_abs_dp":0.0003253115961949031,"top2_margin":0.9668994825333357},"g672":{"argmax":"red","argmax_agree":true,"max_abs_dp":0.0007745623588562012,"mean_abs_dp":0.0005163963263233503,"top2_margin":0.9650477124378085},"g896":{"argmax":"red","argmax_agree":true,"max_abs_dp":0.0004782676696777344,"mean_abs_dp":0.00031885411590337753,"top2_margin":0.9668589923530817}},{"id":"img_09","question_id":"smallest","type":"choice","n_options":3,"native_argmax":"blue","native_top2_margin":0.9375971518456936,"gold":"blue","g256":{"argmax":"blue","argmax_agree":true,"max_abs_dp":0.0071163177490234375,"mean_abs_dp":0.004744221145908038,"top2_margin":0.9492924120277166},"g448":{"argmax":"blue","argmax_agree":true,"max_abs_dp":0.003742694854736328,"mean_abs_dp":0.0024951022739211717,"top2_margin":0.9433693159371614},"g672":{"argmax":"blue","argmax_agree":true,"max_abs_dp":0.004778623580932617,"mean_abs_dp":0.003185742845137914,"top2_margin":0.9458810053765774},"g896":{"argmax":"blue","argmax_agree":true,"max_abs_dp":0.00549846887588501,"mean_abs_dp":0.003665674477815628,"top2_margin":0.9479277543723583}},{"id":"img_09","question_id":"same_size","type":"noul","n_options":2,"native_argmax":"false","native_top2_margin":0.9730345876887441,"gold":"false","g256":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.0016577895730733871,"mean_abs_dp":0.0016577569767832756,"top2_margin":0.9763501016423106},"g448":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.00041456427425146103,"mean_abs_dp":0.0004145572893321514,"top2_margin":0.9738637022674084},"g672":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.0011969208717346191,"mean_abs_dp":0.00119689479470253,"top2_margin":0.970640798099339},"g896":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.0012305974960327148,"mean_abs_dp":0.0012305970303714275,"top2_margin":0.9754957817494869}},{"id":"img_09","question_id":"spread","type":"score","n_options":3,"native_argmax":"2","native_top2_margin":0.3400270342826843,"gold":null,"g256":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.042765915393829346,"mean_abs_dp":0.02851060777902603,"top2_margin":0.4161626994609833},"g448":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.053767383098602295,"mean_abs_dp":0.0358449270327886,"top2_margin":0.4367528706789017},"g672":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.03668004274368286,"mean_abs_dp":0.02445334444443385,"top2_margin":0.4098207950592041},"g896":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.028245925903320312,"mean_abs_dp":0.018830624719460804,"top2_margin":0.3901934027671814}},{"id":"img_10","question_id":"hour","type":"choice","n_options":4,"native_argmax":"3","native_top2_margin":0.9501407779753208,"gold":"3","g256":{"argmax":"3","argmax_agree":true,"max_abs_dp":2.4378299713134766e-05,"mean_abs_dp":1.2201722711324692e-05,"top2_margin":0.9501125644892454},"g448":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.0041261722799390554,"mean_abs_dp":0.0031911387923173606,"top2_margin":0.949541799724102},"g672":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.003992199897766113,"mean_abs_dp":0.0023587867617607117,"top2_margin":0.953407621011138},"g896":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.0031829732470214367,"mean_abs_dp":0.0015914932591840625,"top2_margin":0.9503600187599659}},{"id":"img_10","question_id":"minute_hand_up","type":"noul","n_options":2,"native_argmax":"true","native_top2_margin":0.8968178927898407,"gold":"true","g256":{"argmax":"true","argmax_agree":true,"max_abs_dp":1.9431114196777344e-05,"mean_abs_dp":1.9427388906478882e-05,"top2_margin":0.8967790380120277},"g448":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.004699781537055969,"mean_abs_dp":0.004699774086475372,"top2_margin":0.9062174409627914},"g672":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.01204971969127655,"mean_abs_dp":0.012049667537212372,"top2_margin":0.9209172278642654},"g896":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.005709119141101837,"mean_abs_dp":0.005709115415811539,"top2_margin":0.9082361236214638}},{"id":"img_10","question_id":"confidence","type":"score","n_options":3,"native_argmax":"2","native_top2_margin":0.7686530649662018,"gold":null,"g256":{"argmax":"2","argmax_agree":true,"max_abs_dp":2.6524066925048828e-05,"mean_abs_dp":1.769016186396281e-05,"top2_margin":0.7686230093240738},"g448":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.014450132846832275,"mean_abs_dp":0.009633422829210758,"top2_margin":0.7931901961565018},"g672":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.0074103474617004395,"mean_abs_dp":0.0049402254323164625,"top2_margin":0.7803510203957558},"g896":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.011162042617797852,"mean_abs_dp":0.007441341876983643,"top2_margin":0.7872765585780144}},{"id":"photo_01","question_id":"limes","type":"choice","n_options":6,"native_argmax":"4","native_top2_margin":0.15403717756271362,"gold":"3","g256":{"argmax":"4","argmax_agree":true,"max_abs_dp":0.08159318566322327,"mean_abs_dp":0.027197744231671095,"top2_margin":0.2646184265613556},"g448":{"argmax":"4","argmax_agree":true,"max_abs_dp":0.060935258865356445,"mean_abs_dp":0.02031175931915641,"top2_margin":0.056382983922958374},"g672":{"argmax":"4","argmax_agree":true,"max_abs_dp":0.04243841767311096,"mean_abs_dp":0.01414613394687573,"top2_margin":0.234401136636734},"g896":{"argmax":"3","argmax_agree":false,"max_abs_dp":0.24421268701553345,"mean_abs_dp":0.08269214459384482,"top2_margin":0.3215665817260742}},{"id":"photo_01","question_id":"has_banana","type":"noul","n_options":2,"native_argmax":"false","native_top2_margin":0.9928691615350544,"gold":"false","g256":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.0009750924073159695,"mean_abs_dp":0.0009750823955982924,"top2_margin":0.9909189967438579},"g448":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.00023406906984746456,"mean_abs_dp":0.00023406825494021177,"top2_margin":0.992401025025174},"g672":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.0001748800277709961,"mean_abs_dp":0.00017487804871052504,"top2_margin":0.9925194054376334},"g896":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.00025005568750202656,"mean_abs_dp":0.0002500485861673951,"top2_margin":0.9923690643627197}},{"id":"photo_01","question_id":"kinds","type":"choice","n_options":5,"native_argmax":"4","native_top2_margin":0.4216667264699936,"gold":"4","g256":{"argmax":"5","argmax_agree":false,"max_abs_dp":0.23829680681228638,"mean_abs_dp":0.0964112177491188,"top2_margin":0.00908571481704712},"g448":{"argmax":"4","argmax_agree":true,"max_abs_dp":0.11417073011398315,"mean_abs_dp":0.04566827602684498,"top2_margin":0.23297813534736633},"g672":{"argmax":"4","argmax_agree":true,"max_abs_dp":0.17786270380020142,"mean_abs_dp":0.08206383623182774,"top2_margin":0.06757238507270813},"g896":{"argmax":"4","argmax_agree":true,"max_abs_dp":0.01642024517059326,"mean_abs_dp":0.006568089872598648,"top2_margin":0.39343835413455963}},{"id":"photo_01","question_id":"freshness","type":"score","n_options":3,"native_argmax":"2","native_top2_margin":0.7046183198690414,"gold":null,"g256":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.06854045391082764,"mean_abs_dp":0.045693622436374426,"top2_margin":0.5698036402463913},"g448":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.027018487453460693,"mean_abs_dp":0.018012330401688814,"top2_margin":0.6520294547080994},"g672":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.017767906188964844,"mean_abs_dp":0.01184526194507877,"top2_margin":0.6706755459308624},"g896":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.017619848251342773,"mean_abs_dp":0.01174658148859938,"top2_margin":0.6704102009534836}},{"id":"photo_02","question_id":"animal","type":"choice","n_options":4,"native_argmax":"cat","native_top2_margin":0.949346523731947,"gold":"cat","g256":{"argmax":"cat","argmax_agree":true,"max_abs_dp":0.011899769306182861,"mean_abs_dp":0.005949900019913912,"top2_margin":0.9301810264587402},"g448":{"argmax":"cat","argmax_agree":true,"max_abs_dp":0.004135847091674805,"mean_abs_dp":0.002067934488877654,"top2_margin":0.9437561091035604},"g672":{"argmax":"cat","argmax_agree":true,"max_abs_dp":0.0013308525085449219,"mean_abs_dp":0.0006654388271272182,"top2_margin":0.9470128193497658},"g896":{"argmax":"cat","argmax_agree":true,"max_abs_dp":0.002189815044403076,"mean_abs_dp":0.0010949098505079746,"top2_margin":0.9459417220205069}},{"id":"photo_02","question_id":"eye_color","type":"choice","n_options":4,"native_argmax":"yellow","native_top2_margin":0.9311072155833244,"gold":"yellow","g256":{"argmax":"yellow","argmax_agree":true,"max_abs_dp":0.007892962545156479,"mean_abs_dp":0.004556438536383212,"top2_margin":0.94605927541852},"g448":{"argmax":"yellow","argmax_agree":true,"max_abs_dp":0.004128217697143555,"mean_abs_dp":0.0022157429484650493,"top2_margin":0.9384196568280458},"g672":{"argmax":"yellow","argmax_agree":true,"max_abs_dp":0.006144000217318535,"mean_abs_dp":0.0030719757778570056,"top2_margin":0.9190884623676538},"g896":{"argmax":"yellow","argmax_agree":true,"max_abs_dp":0.00510561466217041,"mean_abs_dp":0.002552809310145676,"top2_margin":0.921521769836545}},{"id":"photo_02","question_id":"background","type":"score","n_options":3,"native_argmax":"0","native_top2_margin":0.6139450818300247,"gold":null,"g256":{"argmax":"0","argmax_agree":true,"max_abs_dp":0.04474562406539917,"mean_abs_dp":0.0298304446041584,"top2_margin":0.5364251285791397},"g448":{"argmax":"0","argmax_agree":true,"max_abs_dp":0.02990555763244629,"mean_abs_dp":0.01993705704808235,"top2_margin":0.5623989552259445},"g672":{"argmax":"0","argmax_agree":true,"max_abs_dp":0.005926787853240967,"mean_abs_dp":0.003951222946246465,"top2_margin":0.6024542152881622},"g896":{"argmax":"0","argmax_agree":true,"max_abs_dp":0.0249178409576416,"mean_abs_dp":0.016611885279417038,"top2_margin":0.5695119053125381}},{"id":"photo_02","question_id":"striped","type":"noul","n_options":2,"native_argmax":"true","native_top2_margin":0.8699666932225227,"gold":"true","g256":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.00871884822845459,"mean_abs_dp":0.008718837052583694,"top2_margin":0.8525290191173553},"g448":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.0031055212020874023,"mean_abs_dp":0.00310550257563591,"top2_margin":0.8637556880712509},"g672":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.00450444221496582,"mean_abs_dp":0.004504438489675522,"top2_margin":0.8609578162431717},"g896":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.006097033619880676,"mean_abs_dp":0.006097026169300079,"top2_margin":0.8577726408839226}},{"id":"photo_03","question_id":"snow_visible","type":"noul","n_options":2,"native_argmax":"true","native_top2_margin":0.8728517517447472,"gold":"true","g256":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.016593359410762787,"mean_abs_dp":0.01659334823489189,"top2_margin":0.8396650552749634},"g448":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.014888554811477661,"mean_abs_dp":0.014888539910316467,"top2_margin":0.8430746719241142},"g672":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.004235178232192993,"mean_abs_dp":0.004235163331031799,"top2_margin":0.8643814250826836},"g896":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.003480084240436554,"mean_abs_dp":0.0034800805151462555,"top2_margin":0.8658915907144547}},{"id":"photo_03","question_id":"reflection","type":"noul","n_options":2,"native_argmax":"true","native_top2_margin":0.911050520837307,"gold":"true","g256":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.0037481188774108887,"mean_abs_dp":0.0037480704486370087,"top2_margin":0.903554379940033},"g448":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.005352497100830078,"mean_abs_dp":0.00535246729850769,"top2_margin":0.9003455862402916},"g672":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.0036435723304748535,"mean_abs_dp":0.0036435648798942566,"top2_margin":0.9037633910775185},"g896":{"argmax":"true","argmax_agree":true,"max_abs_dp":0.002313472330570221,"mean_abs_dp":0.0023134537041187286,"top2_margin":0.9156774282455444}},{"id":"photo_03","question_id":"sky","type":"choice","n_options":4,"native_argmax":"overcast","native_top2_margin":0.9153185542672873,"gold":"overcast","g256":{"argmax":"overcast","argmax_agree":true,"max_abs_dp":0.006427764892578125,"mean_abs_dp":0.003213893622159958,"top2_margin":0.905474217608571},"g448":{"argmax":"overcast","argmax_agree":true,"max_abs_dp":0.0014131180942058563,"mean_abs_dp":0.0007611699402332306,"top2_margin":0.912934297695756},"g672":{"argmax":"overcast","argmax_agree":true,"max_abs_dp":0.005598843097686768,"mean_abs_dp":0.002799420151859522,"top2_margin":0.9071989953517914},"g896":{"argmax":"overcast","argmax_agree":true,"max_abs_dp":0.0005388744175434113,"mean_abs_dp":0.0002694521099328995,"top2_margin":0.9160570446401834}},{"id":"photo_03","question_id":"wildness","type":"score","n_options":4,"native_argmax":"3","native_top2_margin":0.8406233042478561,"gold":null,"g256":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.0074759721755981445,"mean_abs_dp":0.0037379784043878317,"top2_margin":0.8537349253892899},"g448":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.01439046859741211,"mean_abs_dp":0.0071952566504478455,"top2_margin":0.8666355535387993},"g672":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.010815203189849854,"mean_abs_dp":0.005407628603279591,"top2_margin":0.8608631938695908},"g896":{"argmax":"3","argmax_agree":true,"max_abs_dp":0.006258834153413773,"mean_abs_dp":0.0031294168438762426,"top2_margin":0.8525673486292362}},{"id":"photo_04","question_id":"drink","type":"choice","n_options":4,"native_argmax":"coffee","native_top2_margin":0.951157646253705,"gold":"coffee","g256":{"argmax":"coffee","argmax_agree":true,"max_abs_dp":0.0021807774901390076,"mean_abs_dp":0.0010903929360210896,"top2_margin":0.9471176210790873},"g448":{"argmax":"coffee","argmax_agree":true,"max_abs_dp":0.0005410313606262207,"mean_abs_dp":0.0004258470144122839,"top2_margin":0.9513880060985684},"g672":{"argmax":"coffee","argmax_agree":true,"max_abs_dp":0.0008720159530639648,"mean_abs_dp":0.00043599936179816723,"top2_margin":0.9500184394419193},"g896":{"argmax":"coffee","argmax_agree":true,"max_abs_dp":0.0011780858039855957,"mean_abs_dp":0.0005890391767024994,"top2_margin":0.9528255499899387}},{"id":"photo_04","question_id":"milk_added","type":"noul","n_options":2,"native_argmax":"false","native_top2_margin":0.9896592232398689,"gold":"false","g256":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.0005285823717713356,"mean_abs_dp":0.0005285483784973621,"top2_margin":0.9886021264828742},"g448":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.0001253378577530384,"mean_abs_dp":0.00012531341053545475,"top2_margin":0.989408596418798},"g672":{"argmax":"false","argmax_agree":true,"max_abs_dp":0.0003050565719604492,"mean_abs_dp":0.0003050505183637142,"top2_margin":0.9902693242765963},"g896":{"argmax":"false","argmax_agree":true,"max_abs_dp":2.9401853680610657e-06,"mean_abs_dp":2.930406481027603e-06,"top2_margin":0.9896650840528309}},{"id":"photo_04","question_id":"cup_color","type":"choice","n_options":4,"native_argmax":"white","native_top2_margin":0.970263154245913,"gold":"white","g256":{"argmax":"white","argmax_agree":true,"max_abs_dp":0.0007053017616271973,"mean_abs_dp":0.0003526633372530341,"top2_margin":0.9695334816351533},"g448":{"argmax":"white","argmax_agree":true,"max_abs_dp":0.00037353718653321266,"mean_abs_dp":0.00018678198102861643,"top2_margin":0.970035164617002},"g672":{"argmax":"white","argmax_agree":true,"max_abs_dp":0.0002104640007019043,"mean_abs_dp":0.00010645389556884766,"top2_margin":0.970055129379034},"g896":{"argmax":"white","argmax_agree":true,"max_abs_dp":0.0007941126823425293,"mean_abs_dp":0.0003970206016674638,"top2_margin":0.9713846365921199}},{"id":"photo_04","question_id":"beans","type":"score","n_options":3,"native_argmax":"2","native_top2_margin":0.9172757267951965,"gold":"2","g256":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.003370460122823715,"mean_abs_dp":0.002247004769742489,"top2_margin":0.9111810363829136},"g448":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.0008522290736436844,"mean_abs_dp":0.0005681396772464117,"top2_margin":0.9172959104180336},"g672":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.0030298829078674316,"mean_abs_dp":0.0020198840647935867,"top2_margin":0.9122439101338387},"g896":{"argmax":"2","argmax_agree":true,"max_abs_dp":0.0007683634757995605,"mean_abs_dp":0.0005122010285655657,"top2_margin":0.9159623123705387}}]},"lm_head_table":{"tensor":"lm_head.weight","shard":"model-00001-of-00004.safetensors","shape":[248320,4096],"stored_dtype":"BF16","read":"raw safetensors bytes, bf16 -> fp32 by a 16-bit shift (exact)","absmax":0.35546875,"min_nonzero_abs":3.4788172342814505e-11,"nonzero_elements":1017118720,"row_norm":{"min":0.37873873114585876,"max":1.5449111461639404,"median":0.9799873232841492},"fp16_rounding":{"max_rel_error_nonzero":1.0,"elements_below_fp16_normal_min":3317193,"elements_flushed_to_zero":1649,"note":"bf16 values in the fp16 normal range are exact in fp16; error only below 6.1e-5"},"sizes_bytes":{"fp16":2034237440,"int8_per_row_fp16_scale":1017615360,"int8_block32_fp16_scale":1080688640},"int8_scheme":"symmetric, codes -127..127, round-to-nearest, scale = absmax/127 stored fp16","lexical_definition":"mean over the option span of lm_head rows (fp64 accumulation here), as joint_schema_model does in fp32","head_terms":{"prior_logit_scale_param":2.296875,"prior_scale":9.943061786014486,"prior_logit_abs_bound":"prior_scale * ||normalize(lex_q) - normalize(lex)||_2 (Cauchy-Schwarz bound on |d prior|)","lexical_projection_rel_l2":"||W_lex (lex_q - lex)|| / ||W_lex lex||, W_lex = option_lexical_projection.weight"},"fixture":{"records":186,"option_spans":978,"distinct_tokens":810},"tables":{"fp16":{"spans":978,"lexical_max_rel_l2":2.9591300252488127e-08,"lexical_mean_rel_l2":2.023760518588689e-08,"lexical_min_cos":0.9999999999999994,"normalized_lexical_max_abs_diff":2.700074482227255e-08,"prior_logit_abs_bound_max":2.9422788200144335e-07,"lexical_projection_max_rel_l2":1.0432354601470357e-08,"rows_used":810,"row_max_rel_l2":6.781715073447765e-08,"row_mean_rel_l2":1.8764380612818362e-08},"int8_per_row":{"spans":978,"lexical_max_rel_l2":0.01075735215552861,"lexical_mean_rel_l2":0.007476277054748238,"lexical_min_cos":0.999942147372955,"normalized_lexical_max_abs_diff":0.0005456810703562236,"prior_logit_abs_bound_max":0.10695391302126007,"lexical_projection_max_rel_l2":0.004143748964430178,"rows_used":810,"row_max_rel_l2":0.028149093411470653,"row_mean_rel_l2":0.008820503117472994},"int8_block32":{"spans":978,"lexical_max_rel_l2":0.005359997823365582,"lexical_mean_rel_l2":0.00453661406330146,"lexical_min_cos":0.9999856351106136,"normalized_lexical_max_abs_diff":0.0003550347011565501,"prior_logit_abs_bound_max":0.0532949769893504,"lexical_projection_max_rel_l2":0.0020943219194032785,"rows_used":810,"row_max_rel_l2":0.006491217125204306,"row_mean_rel_l2":0.005321151466208286}},"seconds":11.015207052230835},"lm_head_table_effect":{"what":"Effect of the lm_head gather-table format on the head's answers, over every oracle run.","runs":214,"distinct_tokens":810,"int8_scheme":"symmetric, codes -127..127, scale = absmax/127 stored fp16 (per row, or per block of 32 along the hidden dim)","tables":{"fp32":{"questions":405,"argmax_agree":405,"max_abs_dp":1.7881393432617188e-07,"worst":"own_t06/text/urgency","mean_of_run_mean_abs_dp":2.7024052858113926e-09,"flips":[]},"fp16":{"questions":405,"argmax_agree":405,"max_abs_dp":1.7881393432617188e-07,"worst":"own_t06/text/urgency","mean_of_run_mean_abs_dp":1.0905705662873906e-08,"flips":[]},"int8_per_row":{"questions":405,"argmax_agree":405,"max_abs_dp":0.00044345855712890625,"worst":"semif_fb73a85d386cf59b0c51/text/decision","mean_of_run_mean_abs_dp":4.8683842892408385e-05,"flips":[]},"int8_block32":{"questions":405,"argmax_agree":405,"max_abs_dp":0.000246584415435791,"worst":"own_j04/text/duration","mean_of_run_mean_abs_dp":2.1721690417797702e-05,"flips":[]}},"seconds":17.41753911972046},"activation_absmax":{"what":"absmax over every fixture run (CPU fp32 oracle) of the residual stream: embed_tokens output, layer-0 input (image rows scattered in), each decoder layer output; pos0 = position 0, rest = positions >= 1; and of the MLP down_proj input per layer; fp16 max = 65504","runs":214,"embed_tokens":{"pos0":{"max":0.0439453125,"run":"own_t01/text"},"overall":{"max":0.462890625,"run":"own_t01/text"}},"layer_input":{"pos0":{"max":0.0439453125,"run":"own_t01/text"},"rest":{"max":39.22116470336914,"run":"photo_02/native"},"overall":{"max":39.22116470336914,"run":"photo_02/native"}},"layers":[{"layer":0,"pos0":{"max":11.826997756958008,"run":"own_t01/text"},"rest":{"max":59.04935073852539,"run":"photo_02/native"},"overall":{"max":59.04935073852539,"run":"photo_02/native"},"mlp_down_in":{"max":8.482263565063477,"run":"own_t01/text"},"argmax_pos_values":[31,36,46,50,54,58,59,60],"argmax_ch_values":[3994]},{"layer":1,"pos0":{"max":15.984521865844727,"run":"own_t01/text"},"rest":{"max":54.96738815307617,"run":"photo_02/native"},"overall":{"max":54.96738815307617,"run":"photo_02/native"},"mlp_down_in":{"max":8.498800277709961,"run":"photo_01/native"},"argmax_pos_values":[31,36,50,60,64,67,74,101],"argmax_ch_values":[3994]},{"layer":2,"pos0":{"max":32.855560302734375,"run":"own_t01/text"},"rest":{"max":45.82366180419922,"run":"photo_02/native"},"overall":{"max":45.82366180419922,"run":"photo_02/native"},"mlp_down_in":{"max":26.48481559753418,"run":"img_01/g256"},"argmax_pos_values":[0,60,64,67,73,98,101,112],"argmax_ch_values":[3994]},{"layer":3,"pos0":{"max":29.763587951660156,"run":"own_t01/text"},"rest":{"max":49.97127151489258,"run":"photo_02/native"},"overall":{"max":49.97127151489258,"run":"photo_02/native"},"mlp_down_in":{"max":14.121868133544922,"run":"img_04/native"},"argmax_pos_values":[31,50,56,60,64,67,73,74],"argmax_ch_values":[3994]},{"layer":4,"pos0":{"max":28.54509735107422,"run":"own_t01/text"},"rest":{"max":45.271053314208984,"run":"photo_02/native"},"overall":{"max":45.271053314208984,"run":"photo_02/native"},"mlp_down_in":{"max":10.464832305908203,"run":"own_t01/text"},"argmax_pos_values":[31,47,50,54,58,60,64,66],"argmax_ch_values":[3994]},{"layer":5,"pos0":{"max":24.843915939331055,"run":"own_t01/text"},"rest":{"max":40.44756317138672,"run":"photo_02/native"},"overall":{"max":40.44756317138672,"run":"photo_02/native"},"mlp_down_in":{"max":21.32605743408203,"run":"img_08/native"},"argmax_pos_values":[1,38,39,50,58,64,67,83],"argmax_ch_values":[3994]},{"layer":6,"pos0":{"max":63.745628356933594,"run":"own_t01/text"},"rest":{"max":41.88509750366211,"run":"photo_03/native"},"overall":{"max":63.745628356933594,"run":"own_t01/text"},"mlp_down_in":{"max":57.755401611328125,"run":"own_t01/text"},"argmax_pos_values":[0],"argmax_ch_values":[3994]},{"layer":7,"pos0":{"max":44.75798416137695,"run":"own_t01/text"},"rest":{"max":41.03350830078125,"run":"own_t01/text"},"overall":{"max":44.75798416137695,"run":"own_t01/text"},"mlp_down_in":{"max":17.09700584411621,"run":"own_t12/text"},"argmax_pos_values":[0],"argmax_ch_values":[3994]},{"layer":8,"pos0":{"max":39.980770111083984,"run":"own_t01/text"},"rest":{"max":42.81202697753906,"run":"own_t01/text"},"overall":{"max":42.81202697753906,"run":"own_t01/text"},"mlp_down_in":{"max":21.963762283325195,"run":"img_04/native"},"argmax_pos_values":[1],"argmax_ch_values":[3994]},{"layer":9,"pos0":{"max":40.00957107543945,"run":"own_t01/text"},"rest":{"max":43.58789825439453,"run":"own_t01/text"},"overall":{"max":43.58789825439453,"run":"own_t01/text"},"mlp_down_in":{"max":22.9985294342041,"run":"own_t15/text"},"argmax_pos_values":[1],"argmax_ch_values":[3994]},{"layer":10,"pos0":{"max":68.33386993408203,"run":"own_t01/text"},"rest":{"max":45.297386169433594,"run":"photo_03/native"},"overall":{"max":68.33386993408203,"run":"own_t01/text"},"mlp_down_in":{"max":54.73016357421875,"run":"own_t14/text"},"argmax_pos_values":[0],"argmax_ch_values":[3456]},{"layer":11,"pos0":{"max":67.96086883544922,"run":"own_t01/text"},"rest":{"max":42.46574020385742,"run":"own_t01/text"},"overall":{"max":67.96086883544922,"run":"own_t01/text"},"mlp_down_in":{"max":26.518157958984375,"run":"photo_01/g256"},"argmax_pos_values":[0],"argmax_ch_values":[3456]},{"layer":12,"pos0":{"max":67.7262191772461,"run":"own_t01/text"},"rest":{"max":45.47061538696289,"run":"photo_01/native"},"overall":{"max":67.7262191772461,"run":"own_t01/text"},"mlp_down_in":{"max":17.56789779663086,"run":"own_t09/text"},"argmax_pos_values":[0],"argmax_ch_values":[3456]},{"layer":13,"pos0":{"max":67.68138885498047,"run":"own_t01/text"},"rest":{"max":47.98067092895508,"run":"photo_01/native"},"overall":{"max":67.68138885498047,"run":"own_t01/text"},"mlp_down_in":{"max":15.257312774658203,"run":"own_t16/text"},"argmax_pos_values":[0],"argmax_ch_values":[3456]},{"layer":14,"pos0":{"max":74.43360137939453,"run":"own_t01/text"},"rest":{"max":49.68189239501953,"run":"photo_03/native"},"overall":{"max":74.43360137939453,"run":"own_t01/text"},"mlp_down_in":{"max":15.859984397888184,"run":"semif_75b470586834df8c568d/text"},"argmax_pos_values":[0],"argmax_ch_values":[3456]},{"layer":15,"pos0":{"max":70.99459075927734,"run":"own_t01/text"},"rest":{"max":49.140472412109375,"run":"photo_01/native"},"overall":{"max":70.99459075927734,"run":"own_t01/text"},"mlp_down_in":{"max":33.500362396240234,"run":"own_j12/text"},"argmax_pos_values":[0],"argmax_ch_values":[3456]},{"layer":16,"pos0":{"max":69.69348907470703,"run":"own_t01/text"},"rest":{"max":46.89350128173828,"run":"photo_01/native"},"overall":{"max":69.69348907470703,"run":"own_t01/text"},"mlp_down_in":{"max":27.938858032226562,"run":"img_07/g448"},"argmax_pos_values":[0],"argmax_ch_values":[3456]},{"layer":17,"pos0":{"max":68.98184204101562,"run":"own_t01/text"},"rest":{"max":46.64482498168945,"run":"img_02/g256"},"overall":{"max":68.98184204101562,"run":"own_t01/text"},"mlp_down_in":{"max":31.13167381286621,"run":"own_t15/text"},"argmax_pos_values":[0],"argmax_ch_values":[3456]},{"layer":18,"pos0":{"max":79.57122802734375,"run":"own_t01/text"},"rest":{"max":54.768760681152344,"run":"own_t01/text"},"overall":{"max":79.57122802734375,"run":"own_t01/text"},"mlp_down_in":{"max":75.80354309082031,"run":"own_t15/text"},"argmax_pos_values":[0],"argmax_ch_values":[3456]},{"layer":19,"pos0":{"max":43.369117736816406,"run":"own_t01/text"},"rest":{"max":65.39250183105469,"run":"photo_01/native"},"overall":{"max":65.39250183105469,"run":"photo_01/native"},"mlp_down_in":{"max":55.10019302368164,"run":"semif_4aa83af95bb69e90bc2d/text"},"argmax_pos_values":[35,137,238,245,274,293,370,383],"argmax_ch_values":[3994]},{"layer":20,"pos0":{"max":32.75110626220703,"run":"own_t01/text"},"rest":{"max":72.14395904541016,"run":"photo_01/native"},"overall":{"max":72.14395904541016,"run":"photo_01/native"},"mlp_down_in":{"max":54.428062438964844,"run":"own_t15/text"},"argmax_pos_values":[35,161,229,238,244,245,274,293],"argmax_ch_values":[3994]},{"layer":21,"pos0":{"max":31.736745834350586,"run":"own_t01/text"},"rest":{"max":77.74151611328125,"run":"photo_01/native"},"overall":{"max":77.74151611328125,"run":"photo_01/native"},"mlp_down_in":{"max":77.58306884765625,"run":"semif_0aa1447680e4f25862bb/text"},"argmax_pos_values":[32,161,190,229,238,245,270,313],"argmax_ch_values":[3994]},{"layer":22,"pos0":{"max":84.9308090209961,"run":"own_t01/text"},"rest":{"max":157.20413208007812,"run":"own_t02/text"},"overall":{"max":157.20413208007812,"run":"own_t02/text"},"mlp_down_in":{"max":85.80168151855469,"run":"semif_90494c76078ffc3c717a/text"},"argmax_pos_values":[2],"argmax_ch_values":[3994]},{"layer":23,"pos0":{"max":87.37782287597656,"run":"own_t01/text"},"rest":{"max":133.64437866210938,"run":"own_t14/text"},"overall":{"max":133.64437866210938,"run":"own_t14/text"},"mlp_down_in":{"max":81.3852767944336,"run":"own_t15/text"},"argmax_pos_values":[2,43,44,46,47,48,49,56],"argmax_ch_values":[3994]},{"layer":24,"pos0":{"max":96.6823501586914,"run":"own_t01/text"},"rest":{"max":136.00892639160156,"run":"img_02/g256"},"overall":{"max":136.00892639160156,"run":"img_02/g256"},"mlp_down_in":{"max":77.64775085449219,"run":"img_02/g448"},"argmax_pos_values":[2,44,46,47,48,56,107,113],"argmax_ch_values":[3994]},{"layer":25,"pos0":{"max":108.96321105957031,"run":"own_t01/text"},"rest":{"max":141.17959594726562,"run":"img_02/g256"},"overall":{"max":141.17959594726562,"run":"img_02/g256"},"mlp_down_in":{"max":99.61998748779297,"run":"img_04/native"},"argmax_pos_values":[2,136,372],"argmax_ch_values":[3994]},{"layer":26,"pos0":{"max":171.514404296875,"run":"own_t01/text"},"rest":{"max":294.0438232421875,"run":"own_t02/text"},"overall":{"max":294.0438232421875,"run":"own_t02/text"},"mlp_down_in":{"max":156.282470703125,"run":"own_t16/text"},"argmax_pos_values":[2],"argmax_ch_values":[3994]},{"layer":27,"pos0":{"max":123.3812484741211,"run":"own_t13/text"},"rest":{"max":169.48886108398438,"run":"own_t05/text"},"overall":{"max":169.48886108398438,"run":"own_t05/text"},"mlp_down_in":{"max":118.36710357666016,"run":"semif_b8091be929b1e489fcc4/text"},"argmax_pos_values":[27,109,124,145,166,196,287,314],"argmax_ch_values":[3994]},{"layer":28,"pos0":{"max":105.39421844482422,"run":"own_t13/text"},"rest":{"max":175.6095428466797,"run":"own_j10/text"},"overall":{"max":175.6095428466797,"run":"own_j10/text"},"mlp_down_in":{"max":131.26409912109375,"run":"img_07/native"},"argmax_pos_values":[27,103,104,109,114,115,118,119],"argmax_ch_values":[3994]},{"layer":29,"pos0":{"max":111.27910614013672,"run":"own_t01/text"},"rest":{"max":181.67556762695312,"run":"own_j10/text"},"overall":{"max":181.67556762695312,"run":"own_j10/text"},"mlp_down_in":{"max":123.97687530517578,"run":"own_t12/text"},"argmax_pos_values":[27,103,104,105,106,107,114,115],"argmax_ch_values":[3994]},{"layer":30,"pos0":{"max":149.03359985351562,"run":"own_t01/text"},"rest":{"max":174.19656372070312,"run":"photo_04/g448"},"overall":{"max":174.19656372070312,"run":"photo_04/g448"},"mlp_down_in":{"max":177.59646606445312,"run":"img_07/native"},"argmax_pos_values":[2,113,127,128,135,145,157,184],"argmax_ch_values":[3994]},{"layer":31,"pos0":{"max":219.40692138671875,"run":"own_t01/text"},"rest":{"max":2673.527587890625,"run":"semif_76c7a79b1ff2363a60c5/text"},"overall":{"max":2673.527587890625,"run":"semif_76c7a79b1ff2363a60c5/text"},"mlp_down_in":{"max":1407.5579833984375,"run":"own_j08/text"},"argmax_pos_values":[57,89,90,91,92,93,94,95],"argmax_ch_values":[3994]}],"final_norm":{"pos0":{"max":41.517337799072266,"run":"own_t01/text"},"rest":{"max":57.03696060180664,"run":"semif_6db17cc22bd656d78558/text"},"overall":{"max":57.03696060180664,"run":"semif_6db17cc22bd656d78558/text"}},"residual_overall_max":{"max":2673.527587890625,"run":"semif_76c7a79b1ff2363a60c5/text"},"residual_rest_max":{"max":2673.527587890625,"run":"semif_76c7a79b1ff2363a60c5/text"},"mlp_down_in_max":{"max":1407.5579833984375,"run":"own_j08/text"},"fp16_max":65504.0},"fp16_torch_crosscheck":{"what":"Round 3 side check: are the graph's few low-cosine hidden rows an fp16-arithmetic effect?","device":"mps","dtype":"float16","S":16,"runs":[{"id":"photo_04","arm":"g448","tokens":634,"seconds":16.93559541698778,"torch_fp16_vs_oracle":{"min_cos":0.8771613519849808,"argmin":134,"positions_below_0.99":1},"graph_vs_oracle":{"min_cos":0.41295601925274195,"argmin":134,"positions_below_0.99":3},"graph_vs_torch_fp16":{"min_cos":0.32121162029618167,"argmin":134,"positions_below_0.99":3,"max_abs_diff":25.1318359375},"graph_lowest_positions":[{"pos":134,"id":248056,"cos_graph_oracle":0.41295601925274195,"cos_torch16_oracle":0.8771613519849808,"cos_graph_torch16":0.32121162029618167},{"pos":161,"id":248056,"cos_graph_oracle":0.8780536422506171,"cos_torch16_oracle":0.9963580081038907,"cos_graph_torch16":0.9031890645489848},{"pos":209,"id":248056,"cos_graph_oracle":0.9700484039418857,"cos_torch16_oracle":0.9994353505253861,"cos_graph_torch16":0.9623890768292929},{"pos":179,"id":248056,"cos_graph_oracle":0.9903291141308894,"cos_torch16_oracle":0.9999654599350062,"cos_graph_torch16":0.9907486604825158},{"pos":621,"id":198,"cos_graph_oracle":0.9938437884860103,"cos_torch16_oracle":0.9997451105990554,"cos_graph_torch16":0.9959938841963906},{"pos":164,"id":248056,"cos_graph_oracle":0.9951394368302585,"cos_torch16_oracle":0.9999522470597273,"cos_graph_torch16":0.9956123975871936}],"torch_lowest_positions":[{"pos":134,"cos_torch16_oracle":0.8771613519849808,"cos_graph_oracle":0.41295601925274195},{"pos":161,"cos_torch16_oracle":0.9963580081038907,"cos_graph_oracle":0.8780536422506171},{"pos":181,"cos_torch16_oracle":0.9992422894437701,"cos_graph_oracle":0.9994253387936666},{"pos":135,"cos_torch16_oracle":0.9993980107693596,"cos_graph_oracle":0.9954075942622735},{"pos":209,"cos_torch16_oracle":0.9994353505253861,"cos_graph_oracle":0.9700484039418857},{"pos":170,"cos_torch16_oracle":0.9994418404656039,"cos_graph_oracle":0.9979084404899091}]},{"id":"img_04","arm":"native","tokens":1262,"seconds":29.501357333996566,"torch_fp16_vs_oracle":{"min_cos":0.9961232178448558,"argmin":610,"positions_below_0.99":0},"graph_vs_oracle":{"min_cos":0.6580536257810397,"argmin":610,"positions_below_0.99":4},"graph_vs_torch_fp16":{"min_cos":0.6269551956455698,"argmin":610,"positions_below_0.99":4,"max_abs_diff":12.359375},"graph_lowest_positions":[{"pos":610,"id":248056,"cos_graph_oracle":0.6580536257810397,"cos_torch16_oracle":0.9961232178448558,"cos_graph_torch16":0.6269551956455698},{"pos":771,"id":248056,"cos_graph_oracle":0.9614217142124386,"cos_torch16_oracle":0.9987281097550258,"cos_graph_torch16":0.9729539740389826},{"pos":347,"id":248056,"cos_graph_oracle":0.9807018991149481,"cos_torch16_oracle":0.9998158922707124,"cos_graph_torch16":0.9768086928553252},{"pos":604,"id":248056,"cos_graph_oracle":0.9863064278079975,"cos_torch16_oracle":0.9997905619849605,"cos_graph_torch16":0.989140892271615},{"pos":515,"id":248056,"cos_graph_oracle":0.9908026524116165,"cos_torch16_oracle":0.9991760659261155,"cos_graph_torch16":0.9953816080449264},{"pos":675,"id":248056,"cos_graph_oracle":0.991825128506709,"cos_torch16_oracle":0.9998749307382191,"cos_graph_torch16":0.9935374701389668}],"torch_lowest_positions":[{"pos":610,"cos_torch16_oracle":0.9961232178448558,"cos_graph_oracle":0.6580536257810397},{"pos":684,"cos_torch16_oracle":0.9977961334506054,"cos_graph_oracle":0.9984076230122034},{"pos":293,"cos_torch16_oracle":0.9979667545320917,"cos_graph_oracle":0.999745589789971},{"pos":387,"cos_torch16_oracle":0.9982732928100072,"cos_graph_oracle":0.9997077107319489},{"pos":47,"cos_torch16_oracle":0.9985241729585541,"cos_graph_oracle":0.9999975087914907},{"pos":771,"cos_torch16_oracle":0.9987281097550258,"cos_graph_oracle":0.9614217142124386}]},{"id":"photo_04","arm":"native","tokens":1038,"seconds":22.631242249975912,"torch_fp16_vs_oracle":{"min_cos":0.9948199231801579,"argmin":424,"positions_below_0.99":0},"graph_vs_oracle":{"min_cos":0.6621676914220574,"argmin":468,"positions_below_0.99":11},"graph_vs_torch_fp16":{"min_cos":0.6643412240601937,"argmin":468,"positions_below_0.99":11,"max_abs_diff":14.15625},"graph_lowest_positions":[{"pos":468,"id":248056,"cos_graph_oracle":0.6621676914220574,"cos_torch16_oracle":0.999864152350946,"cos_graph_torch16":0.6643412240601937},{"pos":521,"id":248056,"cos_graph_oracle":0.8761729727437003,"cos_torch16_oracle":0.999432573394795,"cos_graph_torch16":0.8897114700177382},{"pos":371,"id":248056,"cos_graph_oracle":0.9336298290686496,"cos_torch16_oracle":0.9999922938212521,"cos_graph_torch16":0.9342917295954996},{"pos":526,"id":248056,"cos_graph_oracle":0.958089903850229,"cos_torch16_oracle":0.9997641627781879,"cos_graph_torch16":0.961775605370297},{"pos":616,"id":248056,"cos_graph_oracle":0.977761677508026,"cos_torch16_oracle":0.9999870325224279,"cos_graph_torch16":0.9785384403354992},{"pos":522,"id":248056,"cos_graph_oracle":0.9788738311758954,"cos_torch16_oracle":0.9997994871523049,"cos_graph_torch16":0.9824058491040999}],"torch_lowest_positions":[{"pos":424,"cos_torch16_oracle":0.9948199231801579,"cos_graph_oracle":0.9994591401256008},{"pos":558,"cos_torch16_oracle":0.9976695490230688,"cos_graph_oracle":0.9958505607303758},{"pos":629,"cos_torch16_oracle":0.9978772800435681,"cos_graph_oracle":0.9983733815929668},{"pos":272,"cos_torch16_oracle":0.998470846304704,"cos_graph_oracle":0.9886738714531376},{"pos":138,"cos_torch16_oracle":0.9989809848908074,"cos_graph_oracle":0.9899661110870527},{"pos":591,"cos_torch16_oracle":0.999102403831649,"cos_graph_oracle":0.9910054685118715}]},{"id":"own_j08","arm":"text","tokens":473,"seconds":10.171503874997143,"torch_fp16_vs_oracle":{"min_cos":0.9972208090859996,"argmin":323,"positions_below_0.99":0},"graph_vs_oracle":{"min_cos":0.9484729535968323,"argmin":323,"positions_below_0.99":1},"graph_vs_torch_fp16":{"min_cos":0.9664798382587706,"argmin":323,"positions_below_0.99":1,"max_abs_diff":4.00146484375},"graph_lowest_positions":[{"pos":323,"id":25,"cos_graph_oracle":0.9484729535968323,"cos_torch16_oracle":0.9972208090859996,"cos_graph_torch16":0.9664798382587706},{"pos":97,"id":1288,"cos_graph_oracle":0.9979384567180924,"cos_torch16_oracle":0.9999895713346478,"cos_graph_torch16":0.9977239956669179},{"pos":324,"id":220,"cos_graph_oracle":0.9979503790752271,"cos_torch16_oracle":0.9998365790357087,"cos_graph_torch16":0.9989163227724591},{"pos":262,"id":220,"cos_graph_oracle":0.9993964577804354,"cos_torch16_oracle":0.9999369920538627,"cos_graph_torch16":0.999021945426329},{"pos":400,"id":25,"cos_graph_oracle":0.999484731472815,"cos_torch16_oracle":0.9999983328550511,"cos_graph_torch16":0.9994489886711383},{"pos":117,"id":25,"cos_graph_oracle":0.9995248591306228,"cos_torch16_oracle":0.9999831185419297,"cos_graph_torch16":0.999665828895086}],"torch_lowest_positions":[{"pos":323,"cos_torch16_oracle":0.9972208090859996,"cos_graph_oracle":0.9484729535968323},{"pos":324,"cos_torch16_oracle":0.9998365790357087,"cos_graph_oracle":0.9979503790752271},{"pos":297,"cos_torch16_oracle":0.9998936451768539,"cos_graph_oracle":0.9999773972618359},{"pos":262,"cos_torch16_oracle":0.9999369920538627,"cos_graph_oracle":0.9993964577804354},{"pos":204,"cos_torch16_oracle":0.9999472217747474,"cos_graph_oracle":0.9999470355745407},{"pos":115,"cos_torch16_oracle":0.9999521351537838,"cos_graph_oracle":0.9996927412404899}]}]}}