{
 "schema": "coreai-scalar-readout-gate/1",
 "mode": "int8lin",
 "bundle": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/exports/system_one_qwen3_5_4b_scorer_decode_int8lin",
 "fixtures": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/oracle-fixtures.json",
 "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
 "max_ctx": 4096,
 "runtime": "coreai Python runtime, AOT h16c GPU asset, SpecializationOptions.default()",
 "source": null,
 "temperature": 1.75,
 "max_len": 384,
 "layout": "State/Question/Option, last token",
 "logits_width": 1,
 "license": "cc-by-nc-4.0",
 "environment": {
  "round": 4,
  "start_epoch": 1790125944.0,
  "deadline_epoch": 1790131344.0,
  "start_utc": "2026-09-22T23:48:21+00:00",
  "deadline_utc": "2026-09-23T02:42:24Z",
  "limit_seconds": 9000,
  "fence": "OPEN",
  "device": "Apple M4 Max, 128 GiB RAM, macOS 27.0 (26A428)",
  "license": "cc-by-nc-4.0",
  "timing": "A: quiet-window GPU throughput; B: correctness only",
  "result": "BLOCKED",
  "phase": "external read-only CPython3.11.13 disappeared; supervisor repair decision pending",
  "updated_at_utc": "2026-09-23T01:08:54.773565+00:00",
  "elapsed_seconds": 4833.77369594574,
  "status": "RUNNING",
  "started_epoch": 1790125944.0,
  "started_utc": "2026-09-23T01:12:24Z",
  "stop_after_seconds": 5400
 },
 "started_at": "2026-09-23T01:18:57+00:00",
 "timing": "contended; correctness only, no performance claim",
 "rows_per_worker": 14,
 "max_row_executions_per_worker": 15,
 "rows": [
  {
   "id": "request-01-q01-option-00",
   "request_id": "request-01",
   "question_id": "request-01-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": 4.328125,
   "scalar_oracle": 4.314200401306152,
   "abs_delta_scalar": 0.013924598693847656,
   "scalar_dtype": "<f2",
   "scalar_bits": "5444",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 28.233943874947727
  },
  {
   "id": "request-01-q01-option-01",
   "request_id": "request-01",
   "question_id": "request-01-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -8.1640625,
   "scalar_oracle": -8.20827865600586,
   "abs_delta_scalar": 0.044216156005859375,
   "scalar_dtype": "<f2",
   "scalar_bits": "15c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 15.978840792085975
  },
  {
   "id": "request-02-q01-option-00",
   "request_id": "request-02",
   "question_id": "request-02-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": 6.19140625,
   "scalar_oracle": 6.207533836364746,
   "abs_delta_scalar": 0.016127586364746094,
   "scalar_dtype": "<f2",
   "scalar_bits": "3146",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 16.093701042002067
  },
  {
   "id": "request-02-q01-option-01",
   "request_id": "request-02",
   "question_id": "request-02-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -9.4921875,
   "scalar_oracle": -9.434723854064941,
   "abs_delta_scalar": 0.057463645935058594,
   "scalar_dtype": "<f2",
   "scalar_bits": "bfc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 8.601522875018418
  },
  {
   "id": "request-03-q01-option-00",
   "request_id": "request-03",
   "question_id": "request-03-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": 6.63671875,
   "scalar_oracle": 6.595953941345215,
   "abs_delta_scalar": 0.040764808654785156,
   "scalar_dtype": "<f2",
   "scalar_bits": "a346",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.805607457994483
  },
  {
   "id": "request-03-q01-option-01",
   "request_id": "request-03",
   "question_id": "request-03-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -8.21875,
   "scalar_oracle": -8.22585678100586,
   "abs_delta_scalar": 0.007106781005859375,
   "scalar_dtype": "<f2",
   "scalar_bits": "1cc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.790163083001971
  },
  {
   "id": "request-04-q01-option-00",
   "request_id": "request-04",
   "question_id": "request-04-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": 5.9609375,
   "scalar_oracle": 5.953964710235596,
   "abs_delta_scalar": 0.006972789764404297,
   "scalar_dtype": "<f2",
   "scalar_bits": "f645",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.788570292061195
  },
  {
   "id": "request-04-q01-option-01",
   "request_id": "request-04",
   "question_id": "request-04-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -10.546875,
   "scalar_oracle": -10.532769203186035,
   "abs_delta_scalar": 0.014105796813964844,
   "scalar_dtype": "<f2",
   "scalar_bits": "46c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.803458750015125
  },
  {
   "id": "request-05-q01-option-00",
   "request_id": "request-05",
   "question_id": "request-05-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": 7.6484375,
   "scalar_oracle": 7.658699989318848,
   "abs_delta_scalar": 0.010262489318847656,
   "scalar_dtype": "<f2",
   "scalar_bits": "a647",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.780388084007427
  },
  {
   "id": "request-05-q01-option-01",
   "request_id": "request-05",
   "question_id": "request-05-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -8.875,
   "scalar_oracle": -8.862298965454102,
   "abs_delta_scalar": 0.012701034545898438,
   "scalar_dtype": "<f2",
   "scalar_bits": "70c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.825818374985829
  },
  {
   "id": "request-06-q01-option-00",
   "request_id": "request-06",
   "question_id": "request-06-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -0.7802734375,
   "scalar_oracle": -0.843937337398529,
   "abs_delta_scalar": 0.06366389989852905,
   "scalar_dtype": "<f2",
   "scalar_bits": "3eba",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 7.825045833014883
  },
  {
   "id": "request-06-q01-option-01",
   "request_id": "request-06",
   "question_id": "request-06-q01",
   "type": "choice",
   "tokens": 384,
   "slot": 383,
   "scalar": -8.34375,
   "scalar_oracle": -8.322973251342773,
   "abs_delta_scalar": 0.020776748657226562,
   "scalar_dtype": "<f2",
   "scalar_bits": "2cc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 8.006930125062354
  },
  {
   "id": "request-07-q01-option-00",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": 4.12109375,
   "scalar_oracle": 4.116079330444336,
   "abs_delta_scalar": 0.0050144195556640625,
   "scalar_dtype": "<f2",
   "scalar_bits": "1f44",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5576650000875816
  },
  {
   "id": "request-07-q01-option-01",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.6875,
   "scalar_oracle": -9.69235897064209,
   "abs_delta_scalar": 0.004858970642089844,
   "scalar_dtype": "<f2",
   "scalar_bits": "d8c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5759510420029983
  },
  {
   "id": "request-07-q01-option-02",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.3125,
   "scalar_oracle": -9.30652904510498,
   "abs_delta_scalar": 0.005970954895019531,
   "scalar_dtype": "<f2",
   "scalar_bits": "a8c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 13.25642637501005
  },
  {
   "id": "request-07-q01-option-03",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -5.73046875,
   "scalar_oracle": -5.676880836486816,
   "abs_delta_scalar": 0.053587913513183594,
   "scalar_dtype": "<f2",
   "scalar_bits": "bbc5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5636381249642
  },
  {
   "id": "request-07-q01-option-04",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -10.5859375,
   "scalar_oracle": -10.599185943603516,
   "abs_delta_scalar": 0.013248443603515625,
   "scalar_dtype": "<f2",
   "scalar_bits": "4bc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5422377919312567
  },
  {
   "id": "request-07-q01-option-05",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.96875,
   "scalar_oracle": -8.972684860229492,
   "abs_delta_scalar": 0.0039348602294921875,
   "scalar_dtype": "<f2",
   "scalar_bits": "7cc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5425575830740854
  },
  {
   "id": "request-07-q01-option-06",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.8125,
   "scalar_oracle": -9.825861930847168,
   "abs_delta_scalar": 0.013361930847167969,
   "scalar_dtype": "<f2",
   "scalar_bits": "e8c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5598199999658391
  },
  {
   "id": "request-07-q01-option-07",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.84765625,
   "scalar_oracle": -6.823195457458496,
   "abs_delta_scalar": 0.024460792541503906,
   "scalar_dtype": "<f2",
   "scalar_bits": "d9c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5392202090006322
  },
  {
   "id": "request-07-q01-option-08",
   "request_id": "request-07",
   "question_id": "request-07-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.8125,
   "scalar_oracle": -6.788428783416748,
   "abs_delta_scalar": 0.024071216583251953,
   "scalar_dtype": "<f2",
   "scalar_bits": "d0c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5402548330603167
  },
  {
   "id": "request-07-q02-option-00",
   "request_id": "request-07",
   "question_id": "request-07-q02",
   "type": "noul",
   "tokens": 28,
   "slot": 27,
   "scalar": 3.21875,
   "scalar_oracle": 3.2154455184936523,
   "abs_delta_scalar": 0.0033044815063476562,
   "scalar_dtype": "<f2",
   "scalar_bits": "7042",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5805649169487879
  },
  {
   "id": "request-07-q02-option-01",
   "request_id": "request-07",
   "question_id": "request-07-q02",
   "type": "noul",
   "tokens": 28,
   "slot": 27,
   "scalar": -1.4541015625,
   "scalar_oracle": -1.4813950061798096,
   "abs_delta_scalar": 0.02729344367980957,
   "scalar_dtype": "<f2",
   "scalar_bits": "d1bd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5829272500704974
  },
  {
   "id": "request-07-q03-option-00",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": -0.2230224609375,
   "scalar_oracle": -0.1996888518333435,
   "abs_delta_scalar": 0.023333609104156494,
   "scalar_dtype": "<f2",
   "scalar_bits": "23b3",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.7841629160102457
  },
  {
   "id": "request-07-q03-option-01",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 1.0546875,
   "scalar_oracle": 1.066279649734497,
   "abs_delta_scalar": 0.01159214973449707,
   "scalar_dtype": "<f2",
   "scalar_bits": "383c",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.7859694580547512
  },
  {
   "id": "request-07-q03-option-02",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 0.98681640625,
   "scalar_oracle": 0.9806243181228638,
   "abs_delta_scalar": 0.0061920881271362305,
   "scalar_dtype": "<f2",
   "scalar_bits": "e53b",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.7839455829234794
  },
  {
   "id": "request-07-q03-option-03",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 0.767578125,
   "scalar_oracle": 0.7489750385284424,
   "abs_delta_scalar": 0.018603086471557617,
   "scalar_dtype": "<f2",
   "scalar_bits": "243a",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.7893001249758527
  },
  {
   "id": "request-07-q03-option-04",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 0.461181640625,
   "scalar_oracle": 0.4373612403869629,
   "abs_delta_scalar": 0.02382040023803711,
   "scalar_dtype": "<f2",
   "scalar_bits": "6137",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.79091724997852
  },
  {
   "id": "request-07-q03-option-05",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 0.72998046875,
   "scalar_oracle": 0.6929409503936768,
   "abs_delta_scalar": 0.03703951835632324,
   "scalar_dtype": "<f2",
   "scalar_bits": "d739",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.622124790912494
  },
  {
   "id": "request-07-q03-option-06",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 0.78857421875,
   "scalar_oracle": 0.7530665397644043,
   "abs_delta_scalar": 0.0355076789855957,
   "scalar_dtype": "<f2",
   "scalar_bits": "4f3a",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.7876862089615315
  },
  {
   "id": "request-07-q03-option-07",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": 0.697265625,
   "scalar_oracle": 0.6613963842391968,
   "abs_delta_scalar": 0.03586924076080322,
   "scalar_dtype": "<f2",
   "scalar_bits": "9439",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.7935234169708565
  },
  {
   "id": "request-07-q03-option-08",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 38,
   "slot": 37,
   "scalar": -0.21142578125,
   "scalar_oracle": -0.23028063774108887,
   "abs_delta_scalar": 0.018854856491088867,
   "scalar_dtype": "<f2",
   "scalar_bits": "c4b2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.794419625075534
  },
  {
   "id": "request-07-q03-option-09",
   "request_id": "request-07",
   "question_id": "request-07-q03",
   "type": "score",
   "tokens": 39,
   "slot": 38,
   "scalar": -1.6865234375,
   "scalar_oracle": -1.719403862953186,
   "abs_delta_scalar": 0.032880425453186035,
   "scalar_dtype": "<f2",
   "scalar_bits": "bfbe",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.8074000419583172
  },
  {
   "id": "request-08-q01-option-00",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": 0.93994140625,
   "scalar_oracle": 0.9249671101570129,
   "abs_delta_scalar": 0.01497429609298706,
   "scalar_dtype": "<f2",
   "scalar_bits": "853b",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5473691670922562
  },
  {
   "id": "request-08-q01-option-01",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.0546875,
   "scalar_oracle": -9.055648803710938,
   "abs_delta_scalar": 0.0009613037109375,
   "scalar_dtype": "<f2",
   "scalar_bits": "87c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5476378330495209
  },
  {
   "id": "request-08-q01-option-02",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.91796875,
   "scalar_oracle": -7.874567985534668,
   "abs_delta_scalar": 0.04340076446533203,
   "scalar_dtype": "<f2",
   "scalar_bits": "ebc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5413437909446657
  },
  {
   "id": "request-08-q01-option-03",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -11.4609375,
   "scalar_oracle": -11.47532844543457,
   "abs_delta_scalar": 0.014390945434570312,
   "scalar_dtype": "<f2",
   "scalar_bits": "bbc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5611719579901546
  },
  {
   "id": "request-08-q01-option-04",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.28515625,
   "scalar_oracle": -7.288543701171875,
   "abs_delta_scalar": 0.003387451171875,
   "scalar_dtype": "<f2",
   "scalar_bits": "49c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5407096659764647
  },
  {
   "id": "request-08-q01-option-05",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.640625,
   "scalar_oracle": -9.6605863571167,
   "abs_delta_scalar": 0.01996135711669922,
   "scalar_dtype": "<f2",
   "scalar_bits": "d2c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5620842499192804
  },
  {
   "id": "request-08-q01-option-06",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.515625,
   "scalar_oracle": -9.511998176574707,
   "abs_delta_scalar": 0.0036268234252929688,
   "scalar_dtype": "<f2",
   "scalar_bits": "c2c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5422627499792725
  },
  {
   "id": "request-08-q01-option-07",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -10.328125,
   "scalar_oracle": -10.34797191619873,
   "abs_delta_scalar": 0.01984691619873047,
   "scalar_dtype": "<f2",
   "scalar_bits": "2ac9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5415689999936149
  },
  {
   "id": "request-08-q01-option-08",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.5859375,
   "scalar_oracle": -9.580318450927734,
   "abs_delta_scalar": 0.005619049072265625,
   "scalar_dtype": "<f2",
   "scalar_bits": "cbc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5400585839524865
  },
  {
   "id": "request-08-q01-option-09",
   "request_id": "request-08",
   "question_id": "request-08-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -10.6796875,
   "scalar_oracle": -10.699100494384766,
   "abs_delta_scalar": 0.019412994384765625,
   "scalar_dtype": "<f2",
   "scalar_bits": "57c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.5710699999472126
  },
  {
   "id": "request-08-q02-option-00",
   "request_id": "request-08",
   "question_id": "request-08-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": 3.56640625,
   "scalar_oracle": 3.5709025859832764,
   "abs_delta_scalar": 0.004496335983276367,
   "scalar_dtype": "<f2",
   "scalar_bits": "2243",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5429547079838812
  },
  {
   "id": "request-08-q02-option-01",
   "request_id": "request-08",
   "question_id": "request-08-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": -1.30859375,
   "scalar_oracle": -1.3289239406585693,
   "abs_delta_scalar": 0.020330190658569336,
   "scalar_dtype": "<f2",
   "scalar_bits": "3cbd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5433100410737097
  },
  {
   "id": "request-08-q03-option-00",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -3.2890625,
   "scalar_oracle": -3.3124825954437256,
   "abs_delta_scalar": 0.023420095443725586,
   "scalar_dtype": "<f2",
   "scalar_bits": "94c2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.8673133329721168
  },
  {
   "id": "request-08-q03-option-01",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.88671875,
   "scalar_oracle": -1.9047918319702148,
   "abs_delta_scalar": 0.018073081970214844,
   "scalar_dtype": "<f2",
   "scalar_bits": "8cbf",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.8735096249729395
  },
  {
   "id": "request-08-q03-option-02",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.4072265625,
   "scalar_oracle": -1.4303022623062134,
   "abs_delta_scalar": 0.02307569980621338,
   "scalar_dtype": "<f2",
   "scalar_bits": "a1bd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.8704549591057003
  },
  {
   "id": "request-08-q03-option-03",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.3671875,
   "scalar_oracle": -1.3922051191329956,
   "abs_delta_scalar": 0.025017619132995605,
   "scalar_dtype": "<f2",
   "scalar_bits": "78bd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.8710623339284211
  },
  {
   "id": "request-08-q03-option-04",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.4599609375,
   "scalar_oracle": -1.48871910572052,
   "abs_delta_scalar": 0.02875816822052002,
   "scalar_dtype": "<f2",
   "scalar_bits": "d7bd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.8714072079164907
  },
  {
   "id": "request-08-q03-option-05",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.1826171875,
   "scalar_oracle": -1.2124919891357422,
   "abs_delta_scalar": 0.029874801635742188,
   "scalar_dtype": "<f2",
   "scalar_bits": "bbbc",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.8777215409791097
  },
  {
   "id": "request-08-q03-option-06",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.25,
   "scalar_oracle": -1.2709437608718872,
   "abs_delta_scalar": 0.020943760871887207,
   "scalar_dtype": "<f2",
   "scalar_bits": "00bd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.8656524170655757
  },
  {
   "id": "request-08-q03-option-07",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.3486328125,
   "scalar_oracle": -1.354970097541809,
   "abs_delta_scalar": 0.006337285041809082,
   "scalar_dtype": "<f2",
   "scalar_bits": "65bd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.8667791669722646
  },
  {
   "id": "request-08-q03-option-08",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 42,
   "slot": 41,
   "scalar": -1.494140625,
   "scalar_oracle": -1.4879703521728516,
   "abs_delta_scalar": 0.0061702728271484375,
   "scalar_dtype": "<f2",
   "scalar_bits": "fabd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.8687871659640223
  },
  {
   "id": "request-08-q03-option-09",
   "request_id": "request-08",
   "question_id": "request-08-q03",
   "type": "score",
   "tokens": 43,
   "slot": 42,
   "scalar": -2.4140625,
   "scalar_oracle": -2.396772861480713,
   "abs_delta_scalar": 0.01728963851928711,
   "scalar_dtype": "<f2",
   "scalar_bits": "d4c0",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.8870229159947485
  },
  {
   "id": "request-09-q01-option-00",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": 0.7626953125,
   "scalar_oracle": 0.7694981694221497,
   "abs_delta_scalar": 0.006802856922149658,
   "scalar_dtype": "<f2",
   "scalar_bits": "1a3a",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5402585410047323
  },
  {
   "id": "request-09-q01-option-01",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.5546875,
   "scalar_oracle": -7.545819282531738,
   "abs_delta_scalar": 0.008868217468261719,
   "scalar_dtype": "<f2",
   "scalar_bits": "8ec7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 3.4584102500230074
  },
  {
   "id": "request-09-q01-option-02",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.28125,
   "scalar_oracle": -8.288448333740234,
   "abs_delta_scalar": 0.007198333740234375,
   "scalar_dtype": "<f2",
   "scalar_bits": "24c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1059247499797493
  },
  {
   "id": "request-09-q01-option-03",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.6171875,
   "scalar_oracle": -8.621004104614258,
   "abs_delta_scalar": 0.0038166046142578125,
   "scalar_dtype": "<f2",
   "scalar_bits": "4fc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0933647500351071
  },
  {
   "id": "request-09-q01-option-04",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.46875,
   "scalar_oracle": -8.477935791015625,
   "abs_delta_scalar": 0.009185791015625,
   "scalar_dtype": "<f2",
   "scalar_bits": "3cc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1383464589016512
  },
  {
   "id": "request-09-q01-option-05",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.92578125,
   "scalar_oracle": -6.928310871124268,
   "abs_delta_scalar": 0.002529621124267578,
   "scalar_dtype": "<f2",
   "scalar_bits": "edc6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0898887080838904
  },
  {
   "id": "request-09-q01-option-06",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.3046875,
   "scalar_oracle": -9.343143463134766,
   "abs_delta_scalar": 0.038455963134765625,
   "scalar_dtype": "<f2",
   "scalar_bits": "a7c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1298550830688328
  },
  {
   "id": "request-09-q01-option-07",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -10.5078125,
   "scalar_oracle": -10.518312454223633,
   "abs_delta_scalar": 0.010499954223632812,
   "scalar_dtype": "<f2",
   "scalar_bits": "41c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1219034580281004
  },
  {
   "id": "request-09-q01-option-08",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -10.6015625,
   "scalar_oracle": -10.605207443237305,
   "abs_delta_scalar": 0.0036449432373046875,
   "scalar_dtype": "<f2",
   "scalar_bits": "4dc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1249543329467997
  },
  {
   "id": "request-09-q01-option-09",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -7.953125,
   "scalar_oracle": -7.95442008972168,
   "abs_delta_scalar": 0.0012950897216796875,
   "scalar_dtype": "<f2",
   "scalar_bits": "f4c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1365885840496048
  },
  {
   "id": "request-09-q01-option-10",
   "request_id": "request-09",
   "question_id": "request-09-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.5859375,
   "scalar_oracle": -8.586217880249023,
   "abs_delta_scalar": 0.0002803802490234375,
   "scalar_dtype": "<f2",
   "scalar_bits": "4bc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1269278330728412
  },
  {
   "id": "request-09-q02-option-00",
   "request_id": "request-09",
   "question_id": "request-09-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": -4.53125,
   "scalar_oracle": -4.542425155639648,
   "abs_delta_scalar": 0.011175155639648438,
   "scalar_dtype": "<f2",
   "scalar_bits": "88c4",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.076729500084184
  },
  {
   "id": "request-09-q02-option-01",
   "request_id": "request-09",
   "question_id": "request-09-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": 2.42578125,
   "scalar_oracle": 2.413087844848633,
   "abs_delta_scalar": 0.012693405151367188,
   "scalar_dtype": "<f2",
   "scalar_bits": "da40",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0843349589267746
  },
  {
   "id": "request-09-q03-option-00",
   "request_id": "request-09",
   "question_id": "request-09-q03",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": -6.80078125,
   "scalar_oracle": -6.80530309677124,
   "abs_delta_scalar": 0.004521846771240234,
   "scalar_dtype": "<f2",
   "scalar_bits": "cdc6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1223064999794587
  },
  {
   "id": "request-09-q03-option-01",
   "request_id": "request-09",
   "question_id": "request-09-q03",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": 2.177734375,
   "scalar_oracle": 2.1344544887542725,
   "abs_delta_scalar": 0.04327988624572754,
   "scalar_dtype": "<f2",
   "scalar_bits": "5b40",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.120337791973725
  },
  {
   "id": "request-10-q01-option-00",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": 0.86083984375,
   "scalar_oracle": 0.8400804996490479,
   "abs_delta_scalar": 0.02075934410095215,
   "scalar_dtype": "<f2",
   "scalar_bits": "e33a",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.5693157920613885
  },
  {
   "id": "request-10-q01-option-01",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.265625,
   "scalar_oracle": -9.282438278198242,
   "abs_delta_scalar": 0.016813278198242188,
   "scalar_dtype": "<f2",
   "scalar_bits": "a2c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0789574169320986
  },
  {
   "id": "request-10-q01-option-02",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.3515625,
   "scalar_oracle": -9.375736236572266,
   "abs_delta_scalar": 0.024173736572265625,
   "scalar_dtype": "<f2",
   "scalar_bits": "adc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0802144579356536
  },
  {
   "id": "request-10-q01-option-03",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -11.4375,
   "scalar_oracle": -11.433075904846191,
   "abs_delta_scalar": 0.004424095153808594,
   "scalar_dtype": "<f2",
   "scalar_bits": "b8c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.07275708403904
  },
  {
   "id": "request-10-q01-option-04",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -11.25,
   "scalar_oracle": -11.280050277709961,
   "abs_delta_scalar": 0.030050277709960938,
   "scalar_dtype": "<f2",
   "scalar_bits": "a0c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.07984470797237
  },
  {
   "id": "request-10-q01-option-05",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -11.7578125,
   "scalar_oracle": -11.762323379516602,
   "abs_delta_scalar": 0.0045108795166015625,
   "scalar_dtype": "<f2",
   "scalar_bits": "e1c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0831135000335053
  },
  {
   "id": "request-10-q01-option-06",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -11.578125,
   "scalar_oracle": -11.585577011108398,
   "abs_delta_scalar": 0.0074520111083984375,
   "scalar_dtype": "<f2",
   "scalar_bits": "cac9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.118770166998729
  },
  {
   "id": "request-10-q01-option-07",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -11.296875,
   "scalar_oracle": -11.320062637329102,
   "abs_delta_scalar": 0.023187637329101562,
   "scalar_dtype": "<f2",
   "scalar_bits": "a6c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0823701660847291
  },
  {
   "id": "request-10-q01-option-08",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.4609375,
   "scalar_oracle": -7.437173843383789,
   "abs_delta_scalar": 0.023763656616210938,
   "scalar_dtype": "<f2",
   "scalar_bits": "76c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1064949169522151
  },
  {
   "id": "request-10-q01-option-09",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.7890625,
   "scalar_oracle": -8.777253150939941,
   "abs_delta_scalar": 0.011809349060058594,
   "scalar_dtype": "<f2",
   "scalar_bits": "65c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.077975874999538
  },
  {
   "id": "request-10-q01-option-10",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -10.1953125,
   "scalar_oracle": -10.206853866577148,
   "abs_delta_scalar": 0.011541366577148438,
   "scalar_dtype": "<f2",
   "scalar_bits": "19c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1316741249756888
  },
  {
   "id": "request-10-q01-option-11",
   "request_id": "request-10",
   "question_id": "request-10-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -11.4140625,
   "scalar_oracle": -11.40426254272461,
   "abs_delta_scalar": 0.009799957275390625,
   "scalar_dtype": "<f2",
   "scalar_bits": "b5c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1187094170600176
  },
  {
   "id": "request-10-q02-option-00",
   "request_id": "request-10",
   "question_id": "request-10-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": 2.732421875,
   "scalar_oracle": 2.7351107597351074,
   "abs_delta_scalar": 0.002688884735107422,
   "scalar_dtype": "<f2",
   "scalar_bits": "7741",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0815048329532146
  },
  {
   "id": "request-10-q02-option-01",
   "request_id": "request-10",
   "question_id": "request-10-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": -2.494140625,
   "scalar_oracle": -2.4982452392578125,
   "abs_delta_scalar": 0.0041046142578125,
   "scalar_dtype": "<f2",
   "scalar_bits": "fdc0",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.093403958948329
  },
  {
   "id": "request-10-q03-option-00",
   "request_id": "request-10",
   "question_id": "request-10-q03",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": -2.60546875,
   "scalar_oracle": -2.61518931388855,
   "abs_delta_scalar": 0.009720563888549805,
   "scalar_dtype": "<f2",
   "scalar_bits": "36c1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.7451435000402853
  },
  {
   "id": "request-10-q03-option-01",
   "request_id": "request-10",
   "question_id": "request-10-q03",
   "type": "score",
   "tokens": 31,
   "slot": 30,
   "scalar": 2.798828125,
   "scalar_oracle": 2.7923030853271484,
   "abs_delta_scalar": 0.0065250396728515625,
   "scalar_dtype": "<f2",
   "scalar_bits": "9941",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.2950344999553636
  },
  {
   "id": "request-11-q01-option-00",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": 5.23046875,
   "scalar_oracle": 5.246809482574463,
   "abs_delta_scalar": 0.01634073257446289,
   "scalar_dtype": "<f2",
   "scalar_bits": "3b45",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1306861250195652
  },
  {
   "id": "request-11-q01-option-01",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -6.1796875,
   "scalar_oracle": -6.228903293609619,
   "abs_delta_scalar": 0.04921579360961914,
   "scalar_dtype": "<f2",
   "scalar_bits": "2ec6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1121940829325467
  },
  {
   "id": "request-11-q01-option-02",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -7.18359375,
   "scalar_oracle": -7.158008575439453,
   "abs_delta_scalar": 0.025585174560546875,
   "scalar_dtype": "<f2",
   "scalar_bits": "2fc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1261347910622135
  },
  {
   "id": "request-11-q01-option-03",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -4.40234375,
   "scalar_oracle": -4.403936386108398,
   "abs_delta_scalar": 0.0015926361083984375,
   "scalar_dtype": "<f2",
   "scalar_bits": "67c4",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1180267080198973
  },
  {
   "id": "request-11-q01-option-04",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.56640625,
   "scalar_oracle": -6.593619346618652,
   "abs_delta_scalar": 0.027213096618652344,
   "scalar_dtype": "<f2",
   "scalar_bits": "91c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0795094589702785
  },
  {
   "id": "request-11-q01-option-05",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.4453125,
   "scalar_oracle": -7.444218635559082,
   "abs_delta_scalar": 0.0010938644409179688,
   "scalar_dtype": "<f2",
   "scalar_bits": "72c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0816429170081392
  },
  {
   "id": "request-11-q01-option-06",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.1796875,
   "scalar_oracle": -9.191930770874023,
   "abs_delta_scalar": 0.012243270874023438,
   "scalar_dtype": "<f2",
   "scalar_bits": "97c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0751990840071812
  },
  {
   "id": "request-11-q01-option-07",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.3828125,
   "scalar_oracle": -8.394148826599121,
   "abs_delta_scalar": 0.011336326599121094,
   "scalar_dtype": "<f2",
   "scalar_bits": "31c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0940907499752939
  },
  {
   "id": "request-11-q01-option-08",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.640625,
   "scalar_oracle": -8.630069732666016,
   "abs_delta_scalar": 0.010555267333984375,
   "scalar_dtype": "<f2",
   "scalar_bits": "52c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1295961670111865
  },
  {
   "id": "request-11-q01-option-09",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.53125,
   "scalar_oracle": -9.541899681091309,
   "abs_delta_scalar": 0.010649681091308594,
   "scalar_dtype": "<f2",
   "scalar_bits": "c4c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1174389169318601
  },
  {
   "id": "request-11-q01-option-10",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.98828125,
   "scalar_oracle": -7.013782501220703,
   "abs_delta_scalar": 0.025501251220703125,
   "scalar_dtype": "<f2",
   "scalar_bits": "fdc6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0790207090321928
  },
  {
   "id": "request-11-q01-option-11",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.2265625,
   "scalar_oracle": -9.234328269958496,
   "abs_delta_scalar": 0.007765769958496094,
   "scalar_dtype": "<f2",
   "scalar_bits": "9dc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.14561654208228
  },
  {
   "id": "request-11-q01-option-12",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.3359375,
   "scalar_oracle": -9.323783874511719,
   "abs_delta_scalar": 0.01215362548828125,
   "scalar_dtype": "<f2",
   "scalar_bits": "abc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.6361913749715313
  },
  {
   "id": "request-11-q01-option-13",
   "request_id": "request-11",
   "question_id": "request-11-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.3515625,
   "scalar_oracle": -8.359192848205566,
   "abs_delta_scalar": 0.007630348205566406,
   "scalar_dtype": "<f2",
   "scalar_bits": "2dc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.127640875056386
  },
  {
   "id": "request-11-q02-option-00",
   "request_id": "request-11",
   "question_id": "request-11-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": -3.056640625,
   "scalar_oracle": -3.04154896736145,
   "abs_delta_scalar": 0.015091657638549805,
   "scalar_dtype": "<f2",
   "scalar_bits": "1dc2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0759954580571502
  },
  {
   "id": "request-11-q02-option-01",
   "request_id": "request-11",
   "question_id": "request-11-q02",
   "type": "noul",
   "tokens": 26,
   "slot": 25,
   "scalar": 3.076171875,
   "scalar_oracle": 3.0489227771759033,
   "abs_delta_scalar": 0.02724909782409668,
   "scalar_dtype": "<f2",
   "scalar_bits": "2742",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0819713750388473
  },
  {
   "id": "request-11-q03-option-00",
   "request_id": "request-11",
   "question_id": "request-11-q03",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": -1.96875,
   "scalar_oracle": -1.9961133003234863,
   "abs_delta_scalar": 0.027363300323486328,
   "scalar_dtype": "<f2",
   "scalar_bits": "e0bf",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1357397920219228
  },
  {
   "id": "request-11-q03-option-01",
   "request_id": "request-11",
   "question_id": "request-11-q03",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": -0.794921875,
   "scalar_oracle": -0.7945132851600647,
   "abs_delta_scalar": 0.00040858983993530273,
   "scalar_dtype": "<f2",
   "scalar_bits": "5cba",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1174491670681164
  },
  {
   "id": "request-11-q03-option-02",
   "request_id": "request-11",
   "question_id": "request-11-q03",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": 0.211669921875,
   "scalar_oracle": 0.13405287265777588,
   "abs_delta_scalar": 0.07761704921722412,
   "scalar_dtype": "<f2",
   "scalar_bits": "c632",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.089897166006267
  },
  {
   "id": "request-12-q01-option-00",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": 6.265625,
   "scalar_oracle": 6.2408976554870605,
   "abs_delta_scalar": 0.024727344512939453,
   "scalar_dtype": "<f2",
   "scalar_bits": "4446",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1622014170279726
  },
  {
   "id": "request-12-q01-option-01",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -3.2734375,
   "scalar_oracle": -3.3339803218841553,
   "abs_delta_scalar": 0.06054282188415527,
   "scalar_dtype": "<f2",
   "scalar_bits": "8cc2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1780799169791862
  },
  {
   "id": "request-12-q01-option-02",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -6.76953125,
   "scalar_oracle": -6.770749568939209,
   "abs_delta_scalar": 0.0012183189392089844,
   "scalar_dtype": "<f2",
   "scalar_bits": "c5c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1207258749054745
  },
  {
   "id": "request-12-q01-option-03",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -7.04296875,
   "scalar_oracle": -7.012792587280273,
   "abs_delta_scalar": 0.030176162719726562,
   "scalar_dtype": "<f2",
   "scalar_bits": "0bc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.116859499947168
  },
  {
   "id": "request-12-q01-option-04",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -7.17578125,
   "scalar_oracle": -7.17970609664917,
   "abs_delta_scalar": 0.003924846649169922,
   "scalar_dtype": "<f2",
   "scalar_bits": "2dc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1195257500512525
  },
  {
   "id": "request-12-q01-option-05",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -7.4609375,
   "scalar_oracle": -7.4471611976623535,
   "abs_delta_scalar": 0.013776302337646484,
   "scalar_dtype": "<f2",
   "scalar_bits": "76c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1663933749077842
  },
  {
   "id": "request-12-q01-option-06",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -7.1640625,
   "scalar_oracle": -7.164470195770264,
   "abs_delta_scalar": 0.0004076957702636719,
   "scalar_dtype": "<f2",
   "scalar_bits": "2ac7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1568859999533743
  },
  {
   "id": "request-12-q01-option-07",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -2.654296875,
   "scalar_oracle": -2.6470611095428467,
   "abs_delta_scalar": 0.00723576545715332,
   "scalar_dtype": "<f2",
   "scalar_bits": "4fc1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.640281792031601
  },
  {
   "id": "request-12-q01-option-08",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.59375,
   "scalar_oracle": -6.595610618591309,
   "abs_delta_scalar": 0.0018606185913085938,
   "scalar_dtype": "<f2",
   "scalar_bits": "98c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1373751250794157
  },
  {
   "id": "request-12-q01-option-09",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -7.6796875,
   "scalar_oracle": -7.668771743774414,
   "abs_delta_scalar": 0.010915756225585938,
   "scalar_dtype": "<f2",
   "scalar_bits": "aec7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1198330828920007
  },
  {
   "id": "request-12-q01-option-10",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -8.28125,
   "scalar_oracle": -8.25868034362793,
   "abs_delta_scalar": 0.022569656372070312,
   "scalar_dtype": "<f2",
   "scalar_bits": "24c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1655148330610245
  },
  {
   "id": "request-12-q01-option-11",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -4.83203125,
   "scalar_oracle": -4.801041126251221,
   "abs_delta_scalar": 0.030990123748779297,
   "scalar_dtype": "<f2",
   "scalar_bits": "d5c4",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1566669999156147
  },
  {
   "id": "request-12-q01-option-12",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -8.1015625,
   "scalar_oracle": -8.084135055541992,
   "abs_delta_scalar": 0.017427444458007812,
   "scalar_dtype": "<f2",
   "scalar_bits": "0dc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1748216249980032
  },
  {
   "id": "request-12-q01-option-13",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -7.0078125,
   "scalar_oracle": -7.016900062561035,
   "abs_delta_scalar": 0.009087562561035156,
   "scalar_dtype": "<f2",
   "scalar_bits": "02c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.155517999897711
  },
  {
   "id": "request-12-q01-option-14",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -2.908203125,
   "scalar_oracle": -2.95692777633667,
   "abs_delta_scalar": 0.04872465133666992,
   "scalar_dtype": "<f2",
   "scalar_bits": "d1c1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1719546669628471
  },
  {
   "id": "request-12-q01-option-15",
   "request_id": "request-12",
   "question_id": "request-12-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -3.25390625,
   "scalar_oracle": -3.219796895980835,
   "abs_delta_scalar": 0.03410935401916504,
   "scalar_dtype": "<f2",
   "scalar_bits": "82c2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1707332499790937
  },
  {
   "id": "request-12-q02-option-00",
   "request_id": "request-12",
   "question_id": "request-12-q02",
   "type": "noul",
   "tokens": 28,
   "slot": 27,
   "scalar": 3.04296875,
   "scalar_oracle": 3.03344464302063,
   "abs_delta_scalar": 0.009524106979370117,
   "scalar_dtype": "<f2",
   "scalar_bits": "1642",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.149248749949038
  },
  {
   "id": "request-12-q02-option-01",
   "request_id": "request-12",
   "question_id": "request-12-q02",
   "type": "noul",
   "tokens": 28,
   "slot": 27,
   "scalar": -2.228515625,
   "scalar_oracle": -2.245439291000366,
   "abs_delta_scalar": 0.01692366600036621,
   "scalar_dtype": "<f2",
   "scalar_bits": "75c0",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1844596660230309
  },
  {
   "id": "request-12-q03-option-00",
   "request_id": "request-12",
   "question_id": "request-12-q03",
   "type": "score",
   "tokens": 28,
   "slot": 27,
   "scalar": -2.693359375,
   "scalar_oracle": -2.706667423248291,
   "abs_delta_scalar": 0.013308048248291016,
   "scalar_dtype": "<f2",
   "scalar_bits": "63c1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1660052499501035
  },
  {
   "id": "request-12-q03-option-01",
   "request_id": "request-12",
   "question_id": "request-12-q03",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": -1.3134765625,
   "scalar_oracle": -1.3185935020446777,
   "abs_delta_scalar": 0.005116939544677734,
   "scalar_dtype": "<f2",
   "scalar_bits": "41bd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.2568103750236332
  },
  {
   "id": "request-12-q03-option-02",
   "request_id": "request-12",
   "question_id": "request-12-q03",
   "type": "score",
   "tokens": 28,
   "slot": 27,
   "scalar": 1.490234375,
   "scalar_oracle": 1.510910987854004,
   "abs_delta_scalar": 0.020676612854003906,
   "scalar_dtype": "<f2",
   "scalar_bits": "f63d",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.174165916047059
  },
  {
   "id": "request-13-q01-option-00",
   "request_id": "request-13",
   "question_id": "request-13-q01",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": 4.39453125,
   "scalar_oracle": 4.405121803283691,
   "abs_delta_scalar": 0.010590553283691406,
   "scalar_dtype": "<f2",
   "scalar_bits": "6544",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.7763034999370575
  },
  {
   "id": "request-13-q01-option-01",
   "request_id": "request-13",
   "question_id": "request-13-q01",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": -9.6171875,
   "scalar_oracle": -9.618844985961914,
   "abs_delta_scalar": 0.0016574859619140625,
   "scalar_dtype": "<f2",
   "scalar_bits": "cfc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6364776249974966
  },
  {
   "id": "request-13-q02-option-00",
   "request_id": "request-13",
   "question_id": "request-13-q02",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": 5.375,
   "scalar_oracle": 5.3750176429748535,
   "abs_delta_scalar": 1.7642974853515625e-05,
   "scalar_dtype": "<f2",
   "scalar_bits": "6045",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6462793340906501
  },
  {
   "id": "request-13-q02-option-01",
   "request_id": "request-13",
   "question_id": "request-13-q02",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": -10.1875,
   "scalar_oracle": -10.238785743713379,
   "abs_delta_scalar": 0.051285743713378906,
   "scalar_dtype": "<f2",
   "scalar_bits": "18c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6413107089465484
  },
  {
   "id": "request-13-q03-option-00",
   "request_id": "request-13",
   "question_id": "request-13-q03",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": 7.0546875,
   "scalar_oracle": 7.067084312438965,
   "abs_delta_scalar": 0.012396812438964844,
   "scalar_dtype": "<f2",
   "scalar_bits": "0e47",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6395244169980288
  },
  {
   "id": "request-13-q03-option-01",
   "request_id": "request-13",
   "question_id": "request-13-q03",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": -10.8203125,
   "scalar_oracle": -10.796977996826172,
   "abs_delta_scalar": 0.023334503173828125,
   "scalar_dtype": "<f2",
   "scalar_bits": "69c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6344392499886453
  },
  {
   "id": "request-13-q04-option-00",
   "request_id": "request-13",
   "question_id": "request-13-q04",
   "type": "noul",
   "tokens": 29,
   "slot": 28,
   "scalar": 4.59375,
   "scalar_oracle": 4.599853515625,
   "abs_delta_scalar": 0.006103515625,
   "scalar_dtype": "<f2",
   "scalar_bits": "9844",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5959053330589086
  },
  {
   "id": "request-13-q04-option-01",
   "request_id": "request-13",
   "question_id": "request-13-q04",
   "type": "noul",
   "tokens": 29,
   "slot": 28,
   "scalar": -1.1025390625,
   "scalar_oracle": -1.0872465372085571,
   "abs_delta_scalar": 0.015292525291442871,
   "scalar_dtype": "<f2",
   "scalar_bits": "69bc",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5994644159218296
  },
  {
   "id": "request-13-q05-option-00",
   "request_id": "request-13",
   "question_id": "request-13-q05",
   "type": "score",
   "tokens": 31,
   "slot": 30,
   "scalar": 1.6171875,
   "scalar_oracle": 1.6196714639663696,
   "abs_delta_scalar": 0.002483963966369629,
   "scalar_dtype": "<f2",
   "scalar_bits": "783e",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6345428340137005
  },
  {
   "id": "request-13-q05-option-01",
   "request_id": "request-13",
   "question_id": "request-13-q05",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": 0.30712890625,
   "scalar_oracle": 0.27900874614715576,
   "abs_delta_scalar": 0.02812016010284424,
   "scalar_dtype": "<f2",
   "scalar_bits": "ea34",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6163472080370411
  },
  {
   "id": "request-13-q05-option-02",
   "request_id": "request-13",
   "question_id": "request-13-q05",
   "type": "score",
   "tokens": 31,
   "slot": 30,
   "scalar": -1.2392578125,
   "scalar_oracle": -1.3120677471160889,
   "abs_delta_scalar": 0.07280993461608887,
   "scalar_dtype": "<f2",
   "scalar_bits": "f5bc",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6371415000176057
  },
  {
   "id": "request-13-q05-option-03",
   "request_id": "request-13",
   "question_id": "request-13-q05",
   "type": "score",
   "tokens": 31,
   "slot": 30,
   "scalar": -3.888671875,
   "scalar_oracle": -3.97291898727417,
   "abs_delta_scalar": 0.08424711227416992,
   "scalar_dtype": "<f2",
   "scalar_bits": "c7c3",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6374596250243485
  },
  {
   "id": "request-14-q01-option-00",
   "request_id": "request-14",
   "question_id": "request-14-q01",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": 3.591796875,
   "scalar_oracle": 3.573038101196289,
   "abs_delta_scalar": 0.018758773803710938,
   "scalar_dtype": "<f2",
   "scalar_bits": "2f43",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6386823749635369
  },
  {
   "id": "request-14-q01-option-01",
   "request_id": "request-14",
   "question_id": "request-14-q01",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": -10.9453125,
   "scalar_oracle": -10.914566040039062,
   "abs_delta_scalar": 0.0307464599609375,
   "scalar_dtype": "<f2",
   "scalar_bits": "79c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6354524169582874
  },
  {
   "id": "request-14-q02-option-00",
   "request_id": "request-14",
   "question_id": "request-14-q02",
   "type": "choice",
   "tokens": 32,
   "slot": 31,
   "scalar": -0.489013671875,
   "scalar_oracle": -0.42073899507522583,
   "abs_delta_scalar": 0.06827467679977417,
   "scalar_dtype": "<f2",
   "scalar_bits": "d3b7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.8319387920200825
  },
  {
   "id": "request-14-q02-option-01",
   "request_id": "request-14",
   "question_id": "request-14-q02",
   "type": "choice",
   "tokens": 32,
   "slot": 31,
   "scalar": -9.7109375,
   "scalar_oracle": -9.746484756469727,
   "abs_delta_scalar": 0.03554725646972656,
   "scalar_dtype": "<f2",
   "scalar_bits": "dbc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6582902499940246
  },
  {
   "id": "request-14-q03-option-00",
   "request_id": "request-14",
   "question_id": "request-14-q03",
   "type": "choice",
   "tokens": 32,
   "slot": 31,
   "scalar": 2.412109375,
   "scalar_oracle": 2.417165756225586,
   "abs_delta_scalar": 0.0050563812255859375,
   "scalar_dtype": "<f2",
   "scalar_bits": "d340",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.659038917045109
  },
  {
   "id": "request-14-q03-option-01",
   "request_id": "request-14",
   "question_id": "request-14-q03",
   "type": "choice",
   "tokens": 32,
   "slot": 31,
   "scalar": -10.5234375,
   "scalar_oracle": -10.482742309570312,
   "abs_delta_scalar": 0.0406951904296875,
   "scalar_dtype": "<f2",
   "scalar_bits": "43c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6586649999953806
  },
  {
   "id": "request-14-q04-option-00",
   "request_id": "request-14",
   "question_id": "request-14-q04",
   "type": "noul",
   "tokens": 30,
   "slot": 29,
   "scalar": -1.6494140625,
   "scalar_oracle": -1.5531220436096191,
   "abs_delta_scalar": 0.09629201889038086,
   "scalar_dtype": "<f2",
   "scalar_bits": "99be",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6134705420117825
  },
  {
   "id": "request-14-q04-option-01",
   "request_id": "request-14",
   "question_id": "request-14-q04",
   "type": "noul",
   "tokens": 30,
   "slot": 29,
   "scalar": 1.400390625,
   "scalar_oracle": 1.3508743047714233,
   "abs_delta_scalar": 0.04951632022857666,
   "scalar_dtype": "<f2",
   "scalar_bits": "9a3d",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6168210830073804
  },
  {
   "id": "request-14-q05-option-00",
   "request_id": "request-14",
   "question_id": "request-14-q05",
   "type": "score",
   "tokens": 29,
   "slot": 28,
   "scalar": -7.9453125,
   "scalar_oracle": -7.972799301147461,
   "abs_delta_scalar": 0.027486801147460938,
   "scalar_dtype": "<f2",
   "scalar_bits": "f2c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5956746659940109
  },
  {
   "id": "request-14-q05-option-01",
   "request_id": "request-14",
   "question_id": "request-14-q05",
   "type": "score",
   "tokens": 28,
   "slot": 27,
   "scalar": -3.16015625,
   "scalar_oracle": -3.1746554374694824,
   "abs_delta_scalar": 0.014499187469482422,
   "scalar_dtype": "<f2",
   "scalar_bits": "52c2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5750828330637887
  },
  {
   "id": "request-14-q05-option-02",
   "request_id": "request-14",
   "question_id": "request-14-q05",
   "type": "score",
   "tokens": 28,
   "slot": 27,
   "scalar": -0.0023174285888671875,
   "scalar_oracle": -0.007742464542388916,
   "abs_delta_scalar": 0.0054250359535217285,
   "scalar_dtype": "<f2",
   "scalar_bits": "bf98",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5754311670316383
  },
  {
   "id": "request-14-q05-option-03",
   "request_id": "request-14",
   "question_id": "request-14-q05",
   "type": "score",
   "tokens": 29,
   "slot": 28,
   "scalar": -1.28515625,
   "scalar_oracle": -1.2937160730361938,
   "abs_delta_scalar": 0.008559823036193848,
   "scalar_dtype": "<f2",
   "scalar_bits": "24bd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5988623750163242
  },
  {
   "id": "request-15-q01-option-00",
   "request_id": "request-15",
   "question_id": "request-15-q01",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": -0.61669921875,
   "scalar_oracle": -0.6910357475280762,
   "abs_delta_scalar": 0.07433652877807617,
   "scalar_dtype": "<f2",
   "scalar_bits": "efb8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6362229170044884
  },
  {
   "id": "request-15-q01-option-01",
   "request_id": "request-15",
   "question_id": "request-15-q01",
   "type": "choice",
   "tokens": 30,
   "slot": 29,
   "scalar": -14.59375,
   "scalar_oracle": -14.593600273132324,
   "abs_delta_scalar": 0.00014972686767578125,
   "scalar_dtype": "<f2",
   "scalar_bits": "4ccb",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6160466250730678
  },
  {
   "id": "request-15-q02-option-00",
   "request_id": "request-15",
   "question_id": "request-15-q02",
   "type": "choice",
   "tokens": 30,
   "slot": 29,
   "scalar": 6.65625,
   "scalar_oracle": 6.661491870880127,
   "abs_delta_scalar": 0.005241870880126953,
   "scalar_dtype": "<f2",
   "scalar_bits": "a846",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6157491250196472
  },
  {
   "id": "request-15-q02-option-01",
   "request_id": "request-15",
   "question_id": "request-15-q02",
   "type": "choice",
   "tokens": 30,
   "slot": 29,
   "scalar": -14.5703125,
   "scalar_oracle": -14.577804565429688,
   "abs_delta_scalar": 0.0074920654296875,
   "scalar_dtype": "<f2",
   "scalar_bits": "49cb",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6160072080092505
  },
  {
   "id": "request-15-q03-option-00",
   "request_id": "request-15",
   "question_id": "request-15-q03",
   "type": "choice",
   "tokens": 30,
   "slot": 29,
   "scalar": -1.990234375,
   "scalar_oracle": -2.031851053237915,
   "abs_delta_scalar": 0.04161667823791504,
   "scalar_dtype": "<f2",
   "scalar_bits": "f6bf",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.7991895840968937
  },
  {
   "id": "request-15-q03-option-01",
   "request_id": "request-15",
   "question_id": "request-15-q03",
   "type": "choice",
   "tokens": 31,
   "slot": 30,
   "scalar": -10.9296875,
   "scalar_oracle": -11.027084350585938,
   "abs_delta_scalar": 0.0973968505859375,
   "scalar_dtype": "<f2",
   "scalar_bits": "77c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6427148749353364
  },
  {
   "id": "request-15-q04-option-00",
   "request_id": "request-15",
   "question_id": "request-15-q04",
   "type": "noul",
   "tokens": 28,
   "slot": 27,
   "scalar": -1.669921875,
   "scalar_oracle": -1.7337946891784668,
   "abs_delta_scalar": 0.0638728141784668,
   "scalar_dtype": "<f2",
   "scalar_bits": "aebe",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5768307080725208
  },
  {
   "id": "request-15-q04-option-01",
   "request_id": "request-15",
   "question_id": "request-15-q04",
   "type": "noul",
   "tokens": 28,
   "slot": 27,
   "scalar": 1.3291015625,
   "scalar_oracle": 1.3656667470932007,
   "abs_delta_scalar": 0.036565184593200684,
   "scalar_dtype": "<f2",
   "scalar_bits": "513d",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5789773339638487
  },
  {
   "id": "request-15-q05-option-00",
   "request_id": "request-15",
   "question_id": "request-15-q05",
   "type": "score",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.43359375,
   "scalar_oracle": -6.44263219833374,
   "abs_delta_scalar": 0.009038448333740234,
   "scalar_dtype": "<f2",
   "scalar_bits": "6fc6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5783160409191623
  },
  {
   "id": "request-15-q05-option-01",
   "request_id": "request-15",
   "question_id": "request-15-q05",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": -2.796875,
   "scalar_oracle": -2.8297576904296875,
   "abs_delta_scalar": 0.0328826904296875,
   "scalar_dtype": "<f2",
   "scalar_bits": "98c1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5602630829671398
  },
  {
   "id": "request-15-q05-option-02",
   "request_id": "request-15",
   "question_id": "request-15-q05",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": -0.83154296875,
   "scalar_oracle": -0.8664757013320923,
   "abs_delta_scalar": 0.034932732582092285,
   "scalar_dtype": "<f2",
   "scalar_bits": "a7ba",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.559180208016187
  },
  {
   "id": "request-15-q05-option-03",
   "request_id": "request-15",
   "question_id": "request-15-q05",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": 1.4765625,
   "scalar_oracle": 1.476930022239685,
   "abs_delta_scalar": 0.0003675222396850586,
   "scalar_dtype": "<f2",
   "scalar_bits": "e83d",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5577980829402804
  },
  {
   "id": "request-15-q05-option-04",
   "request_id": "request-15",
   "question_id": "request-15-q05",
   "type": "score",
   "tokens": 27,
   "slot": 26,
   "scalar": -0.794921875,
   "scalar_oracle": -0.7765030860900879,
   "abs_delta_scalar": 0.01841878890991211,
   "scalar_dtype": "<f2",
   "scalar_bits": "5cba",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5620206670137122
  },
  {
   "id": "request-16-q01-option-00",
   "request_id": "request-16",
   "question_id": "request-16-q01",
   "type": "choice",
   "tokens": 35,
   "slot": 34,
   "scalar": 3.568359375,
   "scalar_oracle": 3.515467405319214,
   "abs_delta_scalar": 0.05289196968078613,
   "scalar_dtype": "<f2",
   "scalar_bits": "2343",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.7185219590319321
  },
  {
   "id": "request-16-q01-option-01",
   "request_id": "request-16",
   "question_id": "request-16-q01",
   "type": "choice",
   "tokens": 34,
   "slot": 33,
   "scalar": -8.1953125,
   "scalar_oracle": -8.229701042175293,
   "abs_delta_scalar": 0.03438854217529297,
   "scalar_dtype": "<f2",
   "scalar_bits": "19c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6981045830762014
  },
  {
   "id": "request-16-q02-option-00",
   "request_id": "request-16",
   "question_id": "request-16-q02",
   "type": "choice",
   "tokens": 33,
   "slot": 32,
   "scalar": 3.326171875,
   "scalar_oracle": 3.330043077468872,
   "abs_delta_scalar": 0.0038712024688720703,
   "scalar_dtype": "<f2",
   "scalar_bits": "a742",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6748981670243666
  },
  {
   "id": "request-16-q02-option-01",
   "request_id": "request-16",
   "question_id": "request-16-q02",
   "type": "choice",
   "tokens": 33,
   "slot": 32,
   "scalar": -5.1171875,
   "scalar_oracle": -5.142807960510254,
   "abs_delta_scalar": 0.025620460510253906,
   "scalar_dtype": "<f2",
   "scalar_bits": "1ec5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6733488750178367
  },
  {
   "id": "request-16-q03-option-00",
   "request_id": "request-16",
   "question_id": "request-16-q03",
   "type": "choice",
   "tokens": 34,
   "slot": 33,
   "scalar": -0.65087890625,
   "scalar_oracle": -0.6904910802841187,
   "abs_delta_scalar": 0.03961217403411865,
   "scalar_dtype": "<f2",
   "scalar_bits": "35b9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6979127920931205
  },
  {
   "id": "request-16-q03-option-01",
   "request_id": "request-16",
   "question_id": "request-16-q03",
   "type": "choice",
   "tokens": 34,
   "slot": 33,
   "scalar": -13.2109375,
   "scalar_oracle": -13.2025728225708,
   "abs_delta_scalar": 0.008364677429199219,
   "scalar_dtype": "<f2",
   "scalar_bits": "9bca",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.9494885420426726
  },
  {
   "id": "request-16-q04-option-00",
   "request_id": "request-16",
   "question_id": "request-16-q04",
   "type": "noul",
   "tokens": 32,
   "slot": 31,
   "scalar": 2.197265625,
   "scalar_oracle": 2.1954517364501953,
   "abs_delta_scalar": 0.0018138885498046875,
   "scalar_dtype": "<f2",
   "scalar_bits": "6540",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.814481541980058
  },
  {
   "id": "request-16-q04-option-01",
   "request_id": "request-16",
   "question_id": "request-16-q04",
   "type": "noul",
   "tokens": 32,
   "slot": 31,
   "scalar": -2.68359375,
   "scalar_oracle": -2.713536024093628,
   "abs_delta_scalar": 0.02994227409362793,
   "scalar_dtype": "<f2",
   "scalar_bits": "5ec1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6579454169841483
  },
  {
   "id": "request-16-q05-option-00",
   "request_id": "request-16",
   "question_id": "request-16-q05",
   "type": "score",
   "tokens": 31,
   "slot": 30,
   "scalar": -2.490234375,
   "scalar_oracle": -2.494438886642456,
   "abs_delta_scalar": 0.004204511642456055,
   "scalar_dtype": "<f2",
   "scalar_bits": "fbc0",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.64113945805002
  },
  {
   "id": "request-16-q05-option-01",
   "request_id": "request-16",
   "question_id": "request-16-q05",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": -0.1806640625,
   "scalar_oracle": -0.20091819763183594,
   "abs_delta_scalar": 0.020254135131835938,
   "scalar_dtype": "<f2",
   "scalar_bits": "c8b1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6187399580376223
  },
  {
   "id": "request-16-q05-option-02",
   "request_id": "request-16",
   "question_id": "request-16-q05",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": -1.2763671875,
   "scalar_oracle": -1.2986819744110107,
   "abs_delta_scalar": 0.022314786911010742,
   "scalar_dtype": "<f2",
   "scalar_bits": "1bbd",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6174192499602214
  },
  {
   "id": "request-16-q05-option-03",
   "request_id": "request-16",
   "question_id": "request-16-q05",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": -3.6640625,
   "scalar_oracle": -3.687429666519165,
   "abs_delta_scalar": 0.02336716651916504,
   "scalar_dtype": "<f2",
   "scalar_bits": "54c3",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6177831249078736
  },
  {
   "id": "request-16-q05-option-04",
   "request_id": "request-16",
   "question_id": "request-16-q05",
   "type": "score",
   "tokens": 30,
   "slot": 29,
   "scalar": -7.6796875,
   "scalar_oracle": -7.6829833984375,
   "abs_delta_scalar": 0.0032958984375,
   "scalar_dtype": "<f2",
   "scalar_bits": "aec7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6200651250546798
  },
  {
   "id": "request-17-q01-option-00",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": 5.546875,
   "scalar_oracle": 5.513814926147461,
   "abs_delta_scalar": 0.03306007385253906,
   "scalar_dtype": "<f2",
   "scalar_bits": "8c45",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5383540000766516
  },
  {
   "id": "request-17-q01-option-01",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.29296875,
   "scalar_oracle": -7.31196403503418,
   "abs_delta_scalar": 0.018995285034179688,
   "scalar_dtype": "<f2",
   "scalar_bits": "4bc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5380529169924557
  },
  {
   "id": "request-17-q01-option-02",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.6171875,
   "scalar_oracle": -7.631340503692627,
   "abs_delta_scalar": 0.014153003692626953,
   "scalar_dtype": "<f2",
   "scalar_bits": "9ec7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5379945409949869
  },
  {
   "id": "request-17-q01-option-03",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.91796875,
   "scalar_oracle": -7.937613487243652,
   "abs_delta_scalar": 0.019644737243652344,
   "scalar_dtype": "<f2",
   "scalar_bits": "ebc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5363502079853788
  },
  {
   "id": "request-17-q01-option-04",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.34375,
   "scalar_oracle": -8.349449157714844,
   "abs_delta_scalar": 0.00569915771484375,
   "scalar_dtype": "<f2",
   "scalar_bits": "2cc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.519089667010121
  },
  {
   "id": "request-17-q01-option-05",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.515625,
   "scalar_oracle": -8.52861499786377,
   "abs_delta_scalar": 0.012989997863769531,
   "scalar_dtype": "<f2",
   "scalar_bits": "42c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5192342079244554
  },
  {
   "id": "request-17-q01-option-06",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.515625,
   "scalar_oracle": -8.547520637512207,
   "abs_delta_scalar": 0.03189563751220703,
   "scalar_dtype": "<f2",
   "scalar_bits": "42c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.148768749902956
  },
  {
   "id": "request-17-q01-option-07",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -6.609375,
   "scalar_oracle": -6.649753093719482,
   "abs_delta_scalar": 0.04037809371948242,
   "scalar_dtype": "<f2",
   "scalar_bits": "9cc6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5211830000625923
  },
  {
   "id": "request-17-q01-option-08",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": 2.87109375,
   "scalar_oracle": 2.8335750102996826,
   "abs_delta_scalar": 0.03751873970031738,
   "scalar_dtype": "<f2",
   "scalar_bits": "be41",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5432739169336855
  },
  {
   "id": "request-17-q01-option-09",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.01953125,
   "scalar_oracle": -5.990054130554199,
   "abs_delta_scalar": 0.02947711944580078,
   "scalar_dtype": "<f2",
   "scalar_bits": "05c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5453538750298321
  },
  {
   "id": "request-17-q01-option-10",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.578125,
   "scalar_oracle": -8.59441089630127,
   "abs_delta_scalar": 0.01628589630126953,
   "scalar_dtype": "<f2",
   "scalar_bits": "4ac8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.522645500022918
  },
  {
   "id": "request-17-q01-option-11",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.8828125,
   "scalar_oracle": -8.899070739746094,
   "abs_delta_scalar": 0.01625823974609375,
   "scalar_dtype": "<f2",
   "scalar_bits": "71c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5419959999853745
  },
  {
   "id": "request-17-q01-option-12",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.84375,
   "scalar_oracle": -8.87210464477539,
   "abs_delta_scalar": 0.028354644775390625,
   "scalar_dtype": "<f2",
   "scalar_bits": "6cc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5186327500268817
  },
  {
   "id": "request-17-q01-option-13",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.9296875,
   "scalar_oracle": -9.984298706054688,
   "abs_delta_scalar": 0.0546112060546875,
   "scalar_dtype": "<f2",
   "scalar_bits": "f7c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5374912499682978
  },
  {
   "id": "request-17-q01-option-14",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.8984375,
   "scalar_oracle": -9.920188903808594,
   "abs_delta_scalar": 0.02175140380859375,
   "scalar_dtype": "<f2",
   "scalar_bits": "f3c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5462715839967132
  },
  {
   "id": "request-17-q01-option-15",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.734375,
   "scalar_oracle": -9.742130279541016,
   "abs_delta_scalar": 0.007755279541015625,
   "scalar_dtype": "<f2",
   "scalar_bits": "dec8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5411979999626055
  },
  {
   "id": "request-17-q01-option-16",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -10.65625,
   "scalar_oracle": -10.690881729125977,
   "abs_delta_scalar": 0.03463172912597656,
   "scalar_dtype": "<f2",
   "scalar_bits": "54c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5583123749820516
  },
  {
   "id": "request-17-q01-option-17",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -10.4453125,
   "scalar_oracle": -10.486421585083008,
   "abs_delta_scalar": 0.04110908508300781,
   "scalar_dtype": "<f2",
   "scalar_bits": "39c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5400415420299396
  },
  {
   "id": "request-17-q01-option-18",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.1640625,
   "scalar_oracle": -9.18571662902832,
   "abs_delta_scalar": 0.021654129028320312,
   "scalar_dtype": "<f2",
   "scalar_bits": "95c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5401158749591559
  },
  {
   "id": "request-17-q01-option-19",
   "request_id": "request-17",
   "question_id": "request-17-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -7.57421875,
   "scalar_oracle": -7.604816436767578,
   "abs_delta_scalar": 0.030597686767578125,
   "scalar_dtype": "<f2",
   "scalar_bits": "93c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5201004999689758
  },
  {
   "id": "request-18-q01-option-00",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -8.0703125,
   "scalar_oracle": -8.074076652526855,
   "abs_delta_scalar": 0.0037641525268554688,
   "scalar_dtype": "<f2",
   "scalar_bits": "09c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 26.255831082933582
  },
  {
   "id": "request-18-q01-option-01",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": 3.806640625,
   "scalar_oracle": 3.807345151901245,
   "abs_delta_scalar": 0.0007045269012451172,
   "scalar_dtype": "<f2",
   "scalar_bits": "9d43",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5863864579005167
  },
  {
   "id": "request-18-q01-option-02",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.81640625,
   "scalar_oracle": -6.854000091552734,
   "abs_delta_scalar": 0.037593841552734375,
   "scalar_dtype": "<f2",
   "scalar_bits": "d1c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6289655420696363
  },
  {
   "id": "request-18-q01-option-03",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -8.1328125,
   "scalar_oracle": -8.165847778320312,
   "abs_delta_scalar": 0.0330352783203125,
   "scalar_dtype": "<f2",
   "scalar_bits": "11c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5857923749135807
  },
  {
   "id": "request-18-q01-option-04",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.90625,
   "scalar_oracle": -8.922479629516602,
   "abs_delta_scalar": 0.016229629516601562,
   "scalar_dtype": "<f2",
   "scalar_bits": "74c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5661013340577483
  },
  {
   "id": "request-18-q01-option-05",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.390625,
   "scalar_oracle": -8.393821716308594,
   "abs_delta_scalar": 0.00319671630859375,
   "scalar_dtype": "<f2",
   "scalar_bits": "32c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5634894169634208
  },
  {
   "id": "request-18-q01-option-06",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.48046875,
   "scalar_oracle": -6.5011138916015625,
   "abs_delta_scalar": 0.0206451416015625,
   "scalar_dtype": "<f2",
   "scalar_bits": "7bc6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5838997500250116
  },
  {
   "id": "request-18-q01-option-07",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -5.98828125,
   "scalar_oracle": -6.00640869140625,
   "abs_delta_scalar": 0.01812744140625,
   "scalar_dtype": "<f2",
   "scalar_bits": "fdc5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5620212909998372
  },
  {
   "id": "request-18-q01-option-08",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -5.34765625,
   "scalar_oracle": -5.335151672363281,
   "abs_delta_scalar": 0.01250457763671875,
   "scalar_dtype": "<f2",
   "scalar_bits": "59c5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5844805829692632
  },
  {
   "id": "request-18-q01-option-09",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.62109375,
   "scalar_oracle": -6.5979790687561035,
   "abs_delta_scalar": 0.023114681243896484,
   "scalar_dtype": "<f2",
   "scalar_bits": "9fc6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.582723917090334
  },
  {
   "id": "request-18-q01-option-10",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.8671875,
   "scalar_oracle": -9.877313613891602,
   "abs_delta_scalar": 0.010126113891601562,
   "scalar_dtype": "<f2",
   "scalar_bits": "efc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5645613330416381
  },
  {
   "id": "request-18-q01-option-11",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -9.421875,
   "scalar_oracle": -9.465591430664062,
   "abs_delta_scalar": 0.0437164306640625,
   "scalar_dtype": "<f2",
   "scalar_bits": "b6c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5806247079744935
  },
  {
   "id": "request-18-q01-option-12",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -9.2421875,
   "scalar_oracle": -9.240874290466309,
   "abs_delta_scalar": 0.0013132095336914062,
   "scalar_dtype": "<f2",
   "scalar_bits": "9fc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5624538339907303
  },
  {
   "id": "request-18-q01-option-13",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -9.8046875,
   "scalar_oracle": -9.860201835632324,
   "abs_delta_scalar": 0.05551433563232422,
   "scalar_dtype": "<f2",
   "scalar_bits": "e7c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5844976250082254
  },
  {
   "id": "request-18-q01-option-14",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -9.6953125,
   "scalar_oracle": -9.681700706481934,
   "abs_delta_scalar": 0.013611793518066406,
   "scalar_dtype": "<f2",
   "scalar_bits": "d9c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.374948124983348
  },
  {
   "id": "request-18-q01-option-15",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -10.8203125,
   "scalar_oracle": -10.806360244750977,
   "abs_delta_scalar": 0.013952255249023438,
   "scalar_dtype": "<f2",
   "scalar_bits": "69c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.587496375082992
  },
  {
   "id": "request-18-q01-option-16",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 29,
   "slot": 28,
   "scalar": -11.6953125,
   "scalar_oracle": -11.715137481689453,
   "abs_delta_scalar": 0.019824981689453125,
   "scalar_dtype": "<f2",
   "scalar_bits": "d9c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6035055420361459
  },
  {
   "id": "request-18-q01-option-17",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -10.6484375,
   "scalar_oracle": -10.642301559448242,
   "abs_delta_scalar": 0.0061359405517578125,
   "scalar_dtype": "<f2",
   "scalar_bits": "53c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.583550333045423
  },
  {
   "id": "request-18-q01-option-18",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -3.96484375,
   "scalar_oracle": -3.9700522422790527,
   "abs_delta_scalar": 0.005208492279052734,
   "scalar_dtype": "<f2",
   "scalar_bits": "eec3",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5860397079959512
  },
  {
   "id": "request-18-q01-option-19",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": 1.8759765625,
   "scalar_oracle": 1.8557240962982178,
   "abs_delta_scalar": 0.020252466201782227,
   "scalar_dtype": "<f2",
   "scalar_bits": "813f",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5648094579810277
  },
  {
   "id": "request-18-q01-option-20",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -5.203125,
   "scalar_oracle": -5.19008207321167,
   "abs_delta_scalar": 0.013042926788330078,
   "scalar_dtype": "<f2",
   "scalar_bits": "34c5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5827544169733301
  },
  {
   "id": "request-18-q01-option-21",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.12109375,
   "scalar_oracle": -6.140894889831543,
   "abs_delta_scalar": 0.01980113983154297,
   "scalar_dtype": "<f2",
   "scalar_bits": "1fc6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5824496248969808
  },
  {
   "id": "request-18-q01-option-22",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 28,
   "slot": 27,
   "scalar": -6.74609375,
   "scalar_oracle": -6.777218341827393,
   "abs_delta_scalar": 0.031124591827392578,
   "scalar_dtype": "<f2",
   "scalar_bits": "bfc6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5832281669136137
  },
  {
   "id": "request-18-q01-option-23",
   "request_id": "request-18",
   "question_id": "request-18-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -8.1484375,
   "scalar_oracle": -8.173320770263672,
   "abs_delta_scalar": 0.024883270263671875,
   "scalar_dtype": "<f2",
   "scalar_bits": "13c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5625455839326605
  },
  {
   "id": "request-19-q01-option-00",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.59765625,
   "scalar_oracle": -7.635459899902344,
   "abs_delta_scalar": 0.03780364990234375,
   "scalar_dtype": "<f2",
   "scalar_bits": "99c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.541937250061892
  },
  {
   "id": "request-19-q01-option-01",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.25,
   "scalar_oracle": -8.273568153381348,
   "abs_delta_scalar": 0.023568153381347656,
   "scalar_dtype": "<f2",
   "scalar_bits": "20c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5425276659661904
  },
  {
   "id": "request-19-q01-option-02",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.59765625,
   "scalar_oracle": -7.611283779144287,
   "abs_delta_scalar": 0.01362752914428711,
   "scalar_dtype": "<f2",
   "scalar_bits": "99c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5429283330449834
  },
  {
   "id": "request-19-q01-option-03",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -0.9609375,
   "scalar_oracle": -1.0317230224609375,
   "abs_delta_scalar": 0.0707855224609375,
   "scalar_dtype": "<f2",
   "scalar_bits": "b0bb",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.545261999941431
  },
  {
   "id": "request-19-q01-option-04",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -9.203125,
   "scalar_oracle": -9.20922565460205,
   "abs_delta_scalar": 0.006100654602050781,
   "scalar_dtype": "<f2",
   "scalar_bits": "9ac8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.6108385829720646
  },
  {
   "id": "request-19-q01-option-05",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.6796875,
   "scalar_oracle": -8.708393096923828,
   "abs_delta_scalar": 0.028705596923828125,
   "scalar_dtype": "<f2",
   "scalar_bits": "57c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5244301250204444
  },
  {
   "id": "request-19-q01-option-06",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -10.09375,
   "scalar_oracle": -10.114972114562988,
   "abs_delta_scalar": 0.02122211456298828,
   "scalar_dtype": "<f2",
   "scalar_bits": "0cc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5450475830584764
  },
  {
   "id": "request-19-q01-option-07",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -9.671875,
   "scalar_oracle": -9.697352409362793,
   "abs_delta_scalar": 0.02547740936279297,
   "scalar_dtype": "<f2",
   "scalar_bits": "d6c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.7684882920002565
  },
  {
   "id": "request-19-q01-option-08",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -5.91015625,
   "scalar_oracle": -5.983921051025391,
   "abs_delta_scalar": 0.07376480102539062,
   "scalar_dtype": "<f2",
   "scalar_bits": "e9c5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.7854871670715511
  },
  {
   "id": "request-19-q01-option-09",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.5859375,
   "scalar_oracle": -9.618802070617676,
   "abs_delta_scalar": 0.03286457061767578,
   "scalar_dtype": "<f2",
   "scalar_bits": "cbc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.6437070840038359
  },
  {
   "id": "request-19-q01-option-10",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -10.515625,
   "scalar_oracle": -10.518702507019043,
   "abs_delta_scalar": 0.0030775070190429688,
   "scalar_dtype": "<f2",
   "scalar_bits": "42c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5225280410377309
  },
  {
   "id": "request-19-q01-option-11",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -10.109375,
   "scalar_oracle": -10.130147933959961,
   "abs_delta_scalar": 0.020772933959960938,
   "scalar_dtype": "<f2",
   "scalar_bits": "0ec9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5445347500499338
  },
  {
   "id": "request-19-q01-option-12",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.046875,
   "scalar_oracle": -8.015636444091797,
   "abs_delta_scalar": 0.031238555908203125,
   "scalar_dtype": "<f2",
   "scalar_bits": "06c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5242211250588298
  },
  {
   "id": "request-19-q01-option-13",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.9140625,
   "scalar_oracle": -9.947691917419434,
   "abs_delta_scalar": 0.033629417419433594,
   "scalar_dtype": "<f2",
   "scalar_bits": "f5c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5490892080124468
  },
  {
   "id": "request-19-q01-option-14",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -10.5859375,
   "scalar_oracle": -10.613463401794434,
   "abs_delta_scalar": 0.027525901794433594,
   "scalar_dtype": "<f2",
   "scalar_bits": "4bc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5497042500646785
  },
  {
   "id": "request-19-q01-option-15",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.6953125,
   "scalar_oracle": -9.693241119384766,
   "abs_delta_scalar": 0.002071380615234375,
   "scalar_dtype": "<f2",
   "scalar_bits": "d9c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.544085624977015
  },
  {
   "id": "request-19-q01-option-16",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 27,
   "slot": 26,
   "scalar": -11.84375,
   "scalar_oracle": -11.859024047851562,
   "abs_delta_scalar": 0.0152740478515625,
   "scalar_dtype": "<f2",
   "scalar_bits": "ecc9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5644161659292877
  },
  {
   "id": "request-19-q01-option-17",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -11.140625,
   "scalar_oracle": -11.160334587097168,
   "abs_delta_scalar": 0.01970958709716797,
   "scalar_dtype": "<f2",
   "scalar_bits": "92c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.5395647910190746
  },
  {
   "id": "request-19-q01-option-18",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -9.34375,
   "scalar_oracle": -9.379319190979004,
   "abs_delta_scalar": 0.035569190979003906,
   "scalar_dtype": "<f2",
   "scalar_bits": "acc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.7725470840232447
  },
  {
   "id": "request-19-q01-option-19",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.171875,
   "scalar_oracle": -8.208106994628906,
   "abs_delta_scalar": 0.03623199462890625,
   "scalar_dtype": "<f2",
   "scalar_bits": "16c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0568256250116974
  },
  {
   "id": "request-19-q01-option-20",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -8.9296875,
   "scalar_oracle": -8.927600860595703,
   "abs_delta_scalar": 0.002086639404296875,
   "scalar_dtype": "<f2",
   "scalar_bits": "77c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0953627079725266
  },
  {
   "id": "request-19-q01-option-21",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.9453125,
   "scalar_oracle": -7.999220848083496,
   "abs_delta_scalar": 0.053908348083496094,
   "scalar_dtype": "<f2",
   "scalar_bits": "f2c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0897414579521865
  },
  {
   "id": "request-19-q01-option-22",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.96484375,
   "scalar_oracle": -6.959987640380859,
   "abs_delta_scalar": 0.004856109619140625,
   "scalar_dtype": "<f2",
   "scalar_bits": "f7c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0886263330467045
  },
  {
   "id": "request-19-q01-option-23",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": 1.26953125,
   "scalar_oracle": 1.2542405128479004,
   "abs_delta_scalar": 0.01529073715209961,
   "scalar_dtype": "<f2",
   "scalar_bits": "143d",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0587976250099018
  },
  {
   "id": "request-19-q01-option-24",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -7.0859375,
   "scalar_oracle": -7.119679927825928,
   "abs_delta_scalar": 0.033742427825927734,
   "scalar_dtype": "<f2",
   "scalar_bits": "16c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0592044580262154
  },
  {
   "id": "request-19-q01-option-25",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.0234375,
   "scalar_oracle": -7.052636623382568,
   "abs_delta_scalar": 0.02919912338256836,
   "scalar_dtype": "<f2",
   "scalar_bits": "06c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1040827080141753
  },
  {
   "id": "request-19-q01-option-26",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.6796875,
   "scalar_oracle": -7.682580471038818,
   "abs_delta_scalar": 0.0028929710388183594,
   "scalar_dtype": "<f2",
   "scalar_bits": "aec7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1011833749944344
  },
  {
   "id": "request-19-q01-option-27",
   "request_id": "request-19",
   "question_id": "request-19-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -7.4765625,
   "scalar_oracle": -7.514122486114502,
   "abs_delta_scalar": 0.03755998611450195,
   "scalar_dtype": "<f2",
   "scalar_bits": "7ac7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.1032665420789272
  },
  {
   "id": "request-20-q01-option-00",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -7.8203125,
   "scalar_oracle": -7.80185079574585,
   "abs_delta_scalar": 0.01846170425415039,
   "scalar_dtype": "<f2",
   "scalar_bits": "d2c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0723036249401048
  },
  {
   "id": "request-20-q01-option-01",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -7.390625,
   "scalar_oracle": -7.406404495239258,
   "abs_delta_scalar": 0.015779495239257812,
   "scalar_dtype": "<f2",
   "scalar_bits": "64c7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0643528329674155
  },
  {
   "id": "request-20-q01-option-02",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.3125,
   "scalar_oracle": -8.328375816345215,
   "abs_delta_scalar": 0.015875816345214844,
   "scalar_dtype": "<f2",
   "scalar_bits": "28c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.061312083969824
  },
  {
   "id": "request-20-q01-option-03",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.3125,
   "scalar_oracle": -8.3722505569458,
   "abs_delta_scalar": 0.05975055694580078,
   "scalar_dtype": "<f2",
   "scalar_bits": "28c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0202210410498083
  },
  {
   "id": "request-20-q01-option-04",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": 2.388671875,
   "scalar_oracle": 2.405498504638672,
   "abs_delta_scalar": 0.016826629638671875,
   "scalar_dtype": "<f2",
   "scalar_bits": "c740",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.5279035840649158
  },
  {
   "id": "request-20-q01-option-05",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -5.38671875,
   "scalar_oracle": -5.341571807861328,
   "abs_delta_scalar": 0.045146942138671875,
   "scalar_dtype": "<f2",
   "scalar_bits": "63c5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0121984999859706
  },
  {
   "id": "request-20-q01-option-06",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.9375,
   "scalar_oracle": -8.924745559692383,
   "abs_delta_scalar": 0.012754440307617188,
   "scalar_dtype": "<f2",
   "scalar_bits": "78c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0467869999120012
  },
  {
   "id": "request-20-q01-option-07",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -6.91015625,
   "scalar_oracle": -6.8783721923828125,
   "abs_delta_scalar": 0.0317840576171875,
   "scalar_dtype": "<f2",
   "scalar_bits": "e9c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0094307089457288
  },
  {
   "id": "request-20-q01-option-08",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.8828125,
   "scalar_oracle": -8.876729011535645,
   "abs_delta_scalar": 0.006083488464355469,
   "scalar_dtype": "<f2",
   "scalar_bits": "71c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.033837667084299
  },
  {
   "id": "request-20-q01-option-09",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.15625,
   "scalar_oracle": -8.154253005981445,
   "abs_delta_scalar": 0.0019969940185546875,
   "scalar_dtype": "<f2",
   "scalar_bits": "14c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0410255839815363
  },
  {
   "id": "request-20-q01-option-10",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -9.2265625,
   "scalar_oracle": -9.206731796264648,
   "abs_delta_scalar": 0.019830703735351562,
   "scalar_dtype": "<f2",
   "scalar_bits": "9dc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.9993104999884963
  },
  {
   "id": "request-20-q01-option-11",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -6.13671875,
   "scalar_oracle": -6.098209381103516,
   "abs_delta_scalar": 0.038509368896484375,
   "scalar_dtype": "<f2",
   "scalar_bits": "23c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.049187625059858
  },
  {
   "id": "request-20-q01-option-12",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -1.23046875,
   "scalar_oracle": -1.189605712890625,
   "abs_delta_scalar": 0.040863037109375,
   "scalar_dtype": "<f2",
   "scalar_bits": "ecbc",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.9975184579379857
  },
  {
   "id": "request-20-q01-option-13",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -9.828125,
   "scalar_oracle": -9.841769218444824,
   "abs_delta_scalar": 0.013644218444824219,
   "scalar_dtype": "<f2",
   "scalar_bits": "eac8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.04355104197748
  },
  {
   "id": "request-20-q01-option-14",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -10.515625,
   "scalar_oracle": -10.507135391235352,
   "abs_delta_scalar": 0.008489608764648438,
   "scalar_dtype": "<f2",
   "scalar_bits": "42c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0441671670414507
  },
  {
   "id": "request-20-q01-option-15",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -10.28125,
   "scalar_oracle": -10.270621299743652,
   "abs_delta_scalar": 0.010628700256347656,
   "scalar_dtype": "<f2",
   "scalar_bits": "24c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0410890000639483
  },
  {
   "id": "request-20-q01-option-16",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 26,
   "slot": 25,
   "scalar": -6.26953125,
   "scalar_oracle": -6.233147144317627,
   "abs_delta_scalar": 0.03638410568237305,
   "scalar_dtype": "<f2",
   "scalar_bits": "45c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0816688330378383
  },
  {
   "id": "request-20-q01-option-17",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -10.765625,
   "scalar_oracle": -10.75239372253418,
   "abs_delta_scalar": 0.013231277465820312,
   "scalar_dtype": "<f2",
   "scalar_bits": "62c9",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0025919590843841
  },
  {
   "id": "request-20-q01-option-18",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.09375,
   "scalar_oracle": -8.108002662658691,
   "abs_delta_scalar": 0.014252662658691406,
   "scalar_dtype": "<f2",
   "scalar_bits": "0cc8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 2.5632803339976817
  },
  {
   "id": "request-20-q01-option-19",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -7.0546875,
   "scalar_oracle": -7.108526229858398,
   "abs_delta_scalar": 0.05383872985839844,
   "scalar_dtype": "<f2",
   "scalar_bits": "0ec7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.9865028749918565
  },
  {
   "id": "request-20-q01-option-20",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -2.525390625,
   "scalar_oracle": -2.5550146102905273,
   "abs_delta_scalar": 0.029623985290527344,
   "scalar_dtype": "<f2",
   "scalar_bits": "0dc1",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0437579579884186
  },
  {
   "id": "request-20-q01-option-21",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -7.37109375,
   "scalar_oracle": -7.36898946762085,
   "abs_delta_scalar": 0.0021042823791503906,
   "scalar_dtype": "<f2",
   "scalar_bits": "5fc7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.046492958907038
  },
  {
   "id": "request-20-q01-option-22",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -6.89453125,
   "scalar_oracle": -6.927596569061279,
   "abs_delta_scalar": 0.0330653190612793,
   "scalar_dtype": "<f2",
   "scalar_bits": "e5c6",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0506179170915857
  },
  {
   "id": "request-20-q01-option-23",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -8.109375,
   "scalar_oracle": -8.134787559509277,
   "abs_delta_scalar": 0.025412559509277344,
   "scalar_dtype": "<f2",
   "scalar_bits": "0ec8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0072323329513893
  },
  {
   "id": "request-20-q01-option-24",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -8.0078125,
   "scalar_oracle": -8.004584312438965,
   "abs_delta_scalar": 0.0032281875610351562,
   "scalar_dtype": "<f2",
   "scalar_bits": "01c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0061959999147803
  },
  {
   "id": "request-20-q01-option-25",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -4.9921875,
   "scalar_oracle": -4.9726715087890625,
   "abs_delta_scalar": 0.0195159912109375,
   "scalar_dtype": "<f2",
   "scalar_bits": "fec4",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0502652080031112
  },
  {
   "id": "request-20-q01-option-26",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -5.60546875,
   "scalar_oracle": -5.6035261154174805,
   "abs_delta_scalar": 0.0019426345825195312,
   "scalar_dtype": "<f2",
   "scalar_bits": "9bc5",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0475528750102967
  },
  {
   "id": "request-20-q01-option-27",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -3.26953125,
   "scalar_oracle": -3.2340099811553955,
   "abs_delta_scalar": 0.03552126884460449,
   "scalar_dtype": "<f2",
   "scalar_bits": "8ac2",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0953721250407398
  },
  {
   "id": "request-20-q01-option-28",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -8.1953125,
   "scalar_oracle": -8.21119499206543,
   "abs_delta_scalar": 0.015882492065429688,
   "scalar_dtype": "<f2",
   "scalar_bits": "19c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0533363330177963
  },
  {
   "id": "request-20-q01-option-29",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": -9.1875,
   "scalar_oracle": -9.236793518066406,
   "abs_delta_scalar": 0.04929351806640625,
   "scalar_dtype": "<f2",
   "scalar_bits": "98c8",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0404980840394273
  },
  {
   "id": "request-20-q01-option-30",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 24,
   "slot": 23,
   "scalar": -7.9296875,
   "scalar_oracle": -7.896141052246094,
   "abs_delta_scalar": 0.03354644775390625,
   "scalar_dtype": "<f2",
   "scalar_bits": "eec7",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 0.9964900000486523
  },
  {
   "id": "request-20-q01-option-31",
   "request_id": "request-20",
   "question_id": "request-20-q01",
   "type": "choice",
   "tokens": 25,
   "slot": 24,
   "scalar": 4.171875,
   "scalar_oracle": 4.17133903503418,
   "abs_delta_scalar": 0.0005359649658203125,
   "scalar_dtype": "<f2",
   "scalar_bits": "2c44",
   "logits_shape": [
    1,
    1,
    1
   ],
   "finite": true,
   "wall_seconds_contended": 1.0507014580070972
  }
 ],
 "questions": [
  {
   "id": "request-01-q01",
   "request_id": "request-01",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-01-q01-option-00",
    "request-01-q01-option-01"
   ],
   "options": [
    "electrical repair",
    "surface cleaning"
   ],
   "nopts": 2,
   "scalars": [
    4.328125,
    -8.1640625
   ],
   "scalars_oracle": [
    4.314200401306152,
    -8.20827865600586
   ],
   "p": [
    0.9992066027922655,
    0.0007933972077346241
   ],
   "p_oracle": [
    0.9992202072785928,
    0.0007797927214071797
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "electrical repair",
   "oracle_selected_option": "electrical repair",
   "oracle_margin": 0.9984404145571857,
   "abs_delta_p": [
    1.360448632736766e-05,
    1.3604486327444421e-05
   ],
   "max_abs_delta_p": 1.3604486327444421e-05,
   "mean_abs_delta_p": 1.360448632740604e-05,
   "finite": true
  },
  {
   "id": "request-02-q01",
   "request_id": "request-02",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-02-q01-option-00",
    "request-02-q01-option-01"
   ],
   "options": [
    "late delivery",
    "wrong color"
   ],
   "nopts": 2,
   "scalars": [
    6.19140625,
    -9.4921875
   ],
   "scalars_oracle": [
    6.207533836364746,
    -9.434723854064941
   ],
   "p": [
    0.9998718336778805,
    0.00012816632211949813
   ],
   "p_oracle": [
    0.9998687706764677,
    0.00013122932353227463
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "late delivery",
   "oracle_selected_option": "late delivery",
   "oracle_margin": 0.9997375413529355,
   "abs_delta_p": [
    3.0630014127286387e-06,
    3.063001412776506e-06
   ],
   "max_abs_delta_p": 3.063001412776506e-06,
   "mean_abs_delta_p": 3.0630014127525724e-06,
   "finite": true
  },
  {
   "id": "request-03-q01",
   "request_id": "request-03",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-03-q01-option-00",
    "request-03-q01-option-01"
   ],
   "options": [
    "a damaged hinge",
    "a missing label"
   ],
   "nopts": 2,
   "scalars": [
    6.63671875,
    -8.21875
   ],
   "scalars_oracle": [
    6.595953941345215,
    -8.22585678100586
   ],
   "p": [
    0.999794290384367,
    0.00020570961563304434
   ],
   "p_oracle": [
    0.9997902964838274,
    0.00020970351617246997
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "a damaged hinge",
   "oracle_selected_option": "a damaged hinge",
   "oracle_margin": 0.999580592967655,
   "abs_delta_p": [
    3.993900539511763e-06,
    3.993900539425623e-06
   ],
   "max_abs_delta_p": 3.993900539511763e-06,
   "mean_abs_delta_p": 3.993900539468693e-06,
   "finite": true
  },
  {
   "id": "request-04-q01",
   "request_id": "request-04",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-04-q01-option-00",
    "request-04-q01-option-01"
   ],
   "options": [
    "the cooling equipment",
    "the floor covering"
   ],
   "nopts": 2,
   "scalars": [
    5.9609375,
    -10.546875
   ],
   "scalars_oracle": [
    5.953964710235596,
    -10.532769203186035
   ],
   "p": [
    0.9999199705404698,
    8.002945953022778e-05
   ],
   "p_oracle": [
    0.9999190008429476,
    8.099915705234445e-05
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "the cooling equipment",
   "oracle_selected_option": "the cooling equipment",
   "oracle_margin": 0.9998380016858952,
   "abs_delta_p": [
    9.696975221684667e-07,
    9.69697522116669e-07
   ],
   "max_abs_delta_p": 9.696975221684667e-07,
   "mean_abs_delta_p": 9.696975221425678e-07,
   "finite": true
  },
  {
   "id": "request-05-q01",
   "request_id": "request-05",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-05-q01-option-00",
    "request-05-q01-option-01"
   ],
   "options": [
    "a duplicate charge",
    "a delivery delay"
   ],
   "nopts": 2,
   "scalars": [
    7.6484375,
    -8.875
   ],
   "scalars_oracle": [
    7.658699989318848,
    -8.862298965454102
   ],
   "p": [
    0.9999206818523142,
    7.931814768584966e-05
   ],
   "p_oracle": [
    0.9999205712578337,
    7.942874216629776e-05
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "a duplicate charge",
   "oracle_selected_option": "a duplicate charge",
   "oracle_margin": 0.9998411425156675,
   "abs_delta_p": [
    1.10594480462467e-07,
    1.1059448044810133e-07
   ],
   "max_abs_delta_p": 1.10594480462467e-07,
   "mean_abs_delta_p": 1.1059448045528416e-07,
   "finite": true
  },
  {
   "id": "request-06-q01",
   "request_id": "request-06",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-06-q01-option-00",
    "request-06-q01-option-01"
   ],
   "options": [
    "stop the water",
    "repaint the ceiling"
   ],
   "nopts": 2,
   "scalars": [
    -0.7802734375,
    -8.34375
   ],
   "scalars_oracle": [
    -0.843937337398529,
    -8.322973251342773
   ],
   "p": [
    0.9869003893996197,
    0.013099610600380264
   ],
   "p_oracle": [
    0.9862617084759727,
    0.0137382915240273
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "stop the water",
   "oracle_selected_option": "stop the water",
   "oracle_margin": 0.9725234169519454,
   "abs_delta_p": [
    0.0006386809236469393,
    0.0006386809236470364
   ],
   "max_abs_delta_p": 0.0006386809236470364,
   "mean_abs_delta_p": 0.0006386809236469879,
   "finite": true
  },
  {
   "id": "request-07-q01",
   "request_id": "request-07",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-07-q01-option-00",
    "request-07-q01-option-01",
    "request-07-q01-option-02",
    "request-07-q01-option-03",
    "request-07-q01-option-04",
    "request-07-q01-option-05",
    "request-07-q01-option-06",
    "request-07-q01-option-07",
    "request-07-q01-option-08"
   ],
   "options": [
    "plants",
    "rocks",
    "stars",
    "poems",
    "roads",
    "music",
    "clothing",
    "numbers",
    "weather"
   ],
   "nopts": 9,
   "scalars": [
    4.12109375,
    -9.6875,
    -9.3125,
    -5.73046875,
    -10.5859375,
    -8.96875,
    -9.8125,
    -6.84765625,
    -6.8125
   ],
   "scalars_oracle": [
    4.116079330444336,
    -9.69235897064209,
    -9.30652904510498,
    -5.676880836486816,
    -10.599185943603516,
    -8.972684860229492,
    -9.825861930847168,
    -6.823195457458496,
    -6.788428783416748
   ],
   "p": [
    0.9906912970950493,
    0.0003707519565651567,
    0.00045935299767690664,
    0.0035570754115400614,
    0.00022188082228678422,
    0.000559054824310197,
    0.00034519334726026,
    0.0018786357273727415,
    0.0019167578179387847
   ],
   "p_oracle": [
    0.9905073080192877,
    0.0003707160297848232,
    0.00046215974250602646,
    0.0036775251265417186,
    0.00022079827667603936,
    0.0005592959160818941,
    0.00034348689011294163,
    0.0019101906994275666,
    0.0019485192995812336
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "plants",
   "oracle_selected_option": "plants",
   "oracle_margin": 0.9868297828927459,
   "abs_delta_p": [
    0.00018398907576167023,
    3.592678033347528e-08,
    2.8067448291198217e-06,
    0.00012044971500165718,
    1.0825456107448582e-06,
    2.410917716970124e-07,
    1.7064571473183457e-06,
    3.155497205482513e-05,
    3.1761481642448956e-05
   ],
   "max_abs_delta_p": 0.00018398907576167023,
   "mean_abs_delta_p": 4.151422339997945e-05,
   "finite": true
  },
  {
   "id": "request-07-q02",
   "request_id": "request-07",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-07-q02-option-00",
    "request-07-q02-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    3.21875,
    -1.4541015625
   ],
   "scalars_oracle": [
    3.2154455184936523,
    -1.4813950061798096
   ],
   "p": [
    0.9352451988533148,
    0.06475480114668508
   ],
   "p_oracle": [
    0.9360704396058132,
    0.06392956039418676
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "yes",
   "oracle_selected_option": "yes",
   "oracle_margin": 0.8721408792116264,
   "abs_delta_p": [
    0.0008252407524983463,
    0.0008252407524983185
   ],
   "max_abs_delta_p": 0.0008252407524983463,
   "mean_abs_delta_p": 0.0008252407524983324,
   "finite": true
  },
  {
   "id": "request-07-q03",
   "request_id": "request-07",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-07-q03-option-00",
    "request-07-q03-option-01",
    "request-07-q03-option-02",
    "request-07-q03-option-03",
    "request-07-q03-option-04",
    "request-07-q03-option-05",
    "request-07-q03-option-06",
    "request-07-q03-option-07",
    "request-07-q03-option-08",
    "request-07-q03-option-09"
   ],
   "options": [
    "level 1",
    "level 2",
    "level 3",
    "level 4",
    "level 5",
    "level 6",
    "level 7",
    "level 8",
    "level 9",
    "level 10"
   ],
   "nopts": 10,
   "scalars": [
    -0.2230224609375,
    1.0546875,
    0.98681640625,
    0.767578125,
    0.461181640625,
    0.72998046875,
    0.78857421875,
    0.697265625,
    -0.21142578125,
    -1.6865234375
   ],
   "scalars_oracle": [
    -0.1996888518333435,
    1.066279649734497,
    0.9806243181228638,
    0.7489750385284424,
    0.4373612403869629,
    0.6929409503936768,
    0.7530665397644043,
    0.6613963842391968,
    -0.23028063774108887,
    -1.719403862953186
   ],
   "p": [
    0.0668905628612553,
    0.13881996412774117,
    0.13353910941541403,
    0.11781497336516934,
    0.0988923283790044,
    0.11531078903673647,
    0.11923700360205998,
    0.11317517052682867,
    0.06733529674798773,
    0.028984801937802986
   ],
   "p_oracle": [
    0.06844557046678758,
    0.1410972557943501,
    0.13435741245143595,
    0.11769923201661937,
    0.09850107499890995,
    0.11399026048335885,
    0.11797473484160344,
    0.11195394081789756,
    0.0672594693782866,
    0.028721048750750485
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "level 2",
   "oracle_selected_option": "level 2",
   "oracle_margin": 0.006739843342914137,
   "abs_delta_p": [
    0.0015550076055322742,
    0.0022772916666089216,
    0.0008183030360219279,
    0.0001157413485499692,
    0.0003912533800944501,
    0.0013205285533776229,
    0.001262268760456542,
    0.0012212297089311125,
    7.582736970113135e-05,
    0.0002637531870525005
   ],
   "max_abs_delta_p": 0.0022772916666089216,
   "mean_abs_delta_p": 0.0009301204616326452,
   "finite": true
  },
  {
   "id": "request-08-q01",
   "request_id": "request-08",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-08-q01-option-00",
    "request-08-q01-option-01",
    "request-08-q01-option-02",
    "request-08-q01-option-03",
    "request-08-q01-option-04",
    "request-08-q01-option-05",
    "request-08-q01-option-06",
    "request-08-q01-option-07",
    "request-08-q01-option-08",
    "request-08-q01-option-09"
   ],
   "options": [
    "lock",
    "roof",
    "window",
    "carpet",
    "wall",
    "drain",
    "lamp",
    "shelf",
    "chair",
    "curtain"
   ],
   "nopts": 10,
   "scalars": [
    0.93994140625,
    -9.0546875,
    -7.91796875,
    -11.4609375,
    -7.28515625,
    -9.640625,
    -9.515625,
    -10.328125,
    -9.5859375,
    -10.6796875
   ],
   "scalars_oracle": [
    0.9249671101570129,
    -9.055648803710938,
    -7.874567985534668,
    -11.47532844543457,
    -7.288543701171875,
    -9.6605863571167,
    -9.511998176574707,
    -10.34797191619873,
    -9.580318450927734,
    -10.699100494384766
   ],
   "p": [
    0.971031652735685,
    0.0032127990904810676,
    0.006151501014341628,
    0.0008123228234390902,
    0.008831309123808445,
    0.0022986479894280478,
    0.002468843175278437,
    0.001551875154060871,
    0.0023716149092534553,
    0.00126943398422394
   ],
   "p_oracle": [
    0.9707076233156018,
    0.0032375478941282032,
    0.0063580361304097265,
    0.0008123224916065292,
    0.00888700915380173,
    0.002291341877189737,
    0.0024943923445487387,
    0.0015470437789951472,
    0.0023988872727527663,
    0.00126579574096568
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "lock",
   "oracle_selected_option": "lock",
   "oracle_margin": 0.9618206141618001,
   "abs_delta_p": [
    0.0003240294200832494,
    2.4748803647135602e-05,
    0.0002065351160680986,
    3.3183256099508784e-10,
    5.570002999328444e-05,
    7.306112238310552e-06,
    2.5549169270301913e-05,
    4.831375065723767e-06,
    2.7272363499311007e-05,
    3.638243258259925e-06
   ],
   "max_abs_delta_p": 0.0003240294200832494,
   "mean_abs_delta_p": 6.796109649562362e-05,
   "finite": true
  },
  {
   "id": "request-08-q02",
   "request_id": "request-08",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-08-q02-option-00",
    "request-08-q02-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    3.56640625,
    -1.30859375
   ],
   "scalars_oracle": [
    3.5709025859832764,
    -1.3289239406585693
   ],
   "p": [
    0.9418989513483157,
    0.05810104865168431
   ],
   "p_oracle": [
    0.942670467195647,
    0.05732953280435297
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "yes",
   "oracle_selected_option": "yes",
   "oracle_margin": 0.885340934391294,
   "abs_delta_p": [
    0.0007715158473312611,
    0.0007715158473313444
   ],
   "max_abs_delta_p": 0.0007715158473313444,
   "mean_abs_delta_p": 0.0007715158473313027,
   "finite": true
  },
  {
   "id": "request-08-q03",
   "request_id": "request-08",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-08-q03-option-00",
    "request-08-q03-option-01",
    "request-08-q03-option-02",
    "request-08-q03-option-03",
    "request-08-q03-option-04",
    "request-08-q03-option-05",
    "request-08-q03-option-06",
    "request-08-q03-option-07",
    "request-08-q03-option-08",
    "request-08-q03-option-09"
   ],
   "options": [
    "level 1",
    "level 2",
    "level 3",
    "level 4",
    "level 5",
    "level 6",
    "level 7",
    "level 8",
    "level 9",
    "level 10"
   ],
   "nopts": 10,
   "scalars": [
    -3.2890625,
    -1.88671875,
    -1.4072265625,
    -1.3671875,
    -1.4599609375,
    -1.1826171875,
    -1.25,
    -1.3486328125,
    -1.494140625,
    -2.4140625
   ],
   "scalars_oracle": [
    -3.3124825954437256,
    -1.9047918319702148,
    -1.4303022623062134,
    -1.3922051191329956,
    -1.48871910572052,
    -1.2124919891357422,
    -1.2709437608718872,
    -1.354970097541809,
    -1.4879703521728516,
    -2.396772861480713
   ],
   "p": [
    0.03844735559601341,
    0.08568083778528952,
    0.11268820304996982,
    0.11529616943900456,
    0.1093431181851565,
    0.12812072124379084,
    0.12328126965086839,
    0.11652512124167493,
    0.1072282308349132,
    0.06338897297331891
   ],
   "p_oracle": [
    0.03829370334805408,
    0.08559956498729611,
    0.11225994259558048,
    0.11473061408817004,
    0.10857444261658075,
    0.12713889005910925,
    0.12296245919395053,
    0.11719791061365697,
    0.10862090712691694,
    0.06462156537068481
   ],
   "argmax": 5,
   "oracle_argmax": 5,
   "argmax_agrees": true,
   "selected_option": "level 6",
   "oracle_selected_option": "level 6",
   "oracle_margin": 0.004176430865158717,
   "abs_delta_p": [
    0.0001536522479593308,
    8.127279799341469e-05,
    0.00042826045438934535,
    0.0005655553508345235,
    0.0007686755685757524,
    0.000981831184681592,
    0.0003188104569178535,
    0.0006727893719820494,
    0.001392676292003736,
    0.0012325923973658948
   ],
   "max_abs_delta_p": 0.001392676292003736,
   "mean_abs_delta_p": 0.0006596116122703493,
   "finite": true
  },
  {
   "id": "request-09-q01",
   "request_id": "request-09",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-09-q01-option-00",
    "request-09-q01-option-01",
    "request-09-q01-option-02",
    "request-09-q01-option-03",
    "request-09-q01-option-04",
    "request-09-q01-option-05",
    "request-09-q01-option-06",
    "request-09-q01-option-07",
    "request-09-q01-option-08",
    "request-09-q01-option-09",
    "request-09-q01-option-10"
   ],
   "options": [
    "rain",
    "snow",
    "fog",
    "heat",
    "frost",
    "wind",
    "dust",
    "smoke",
    "sunshine",
    "hail",
    "drought"
   ],
   "nopts": 11,
   "scalars": [
    0.7626953125,
    -7.5546875,
    -8.28125,
    -8.6171875,
    -8.46875,
    -6.92578125,
    -9.3046875,
    -10.5078125,
    -10.6015625,
    -7.953125,
    -8.5859375
   ],
   "scalars_oracle": [
    0.7694981694221497,
    -7.545819282531738,
    -8.288448333740234,
    -8.621004104614258,
    -8.477935791015625,
    -6.928310871124268,
    -9.343143463134766,
    -10.518312454223633,
    -10.605207443237305,
    -7.95442008972168,
    -8.586217880249023
   ],
   "p": [
    0.9483706159817266,
    0.008182151964925274,
    0.00540203978919264,
    0.004458499771177331,
    0.004853178076426365,
    0.011720391551394505,
    0.0030100454924365284,
    0.0015135459247261776,
    0.0014345966991774674,
    0.006516103802299901,
    0.004538830946517389
   ],
   "p_oracle": [
    0.9486699494562557,
    0.008194399866555504,
    0.005360683684473484,
    0.00443292506160241,
    0.00481055745786062,
    0.011661734510957271,
    0.0029341229642238504,
    0.0014991277682935508,
    0.0014265075320346937,
    0.006488068030313924,
    0.00452192366742885
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "rain",
   "oracle_selected_option": "rain",
   "oracle_margin": 0.9370082149452984,
   "abs_delta_p": [
    0.00029933347452915093,
    1.2247901630230498e-05,
    4.1356104719156295e-05,
    2.5574709574920952e-05,
    4.262061856574479e-05,
    5.8657040437233576e-05,
    7.592252821267799e-05,
    1.4418156432626818e-05,
    8.089167142773766e-06,
    2.803577198597692e-05,
    1.6907279088539638e-05
   ],
   "max_abs_delta_p": 0.00029933347452915093,
   "mean_abs_delta_p": 5.66511593017302e-05,
   "finite": true
  },
  {
   "id": "request-09-q02",
   "request_id": "request-09",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-09-q02-option-00",
    "request-09-q02-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    -4.53125,
    2.42578125
   ],
   "scalars_oracle": [
    -4.542425155639648,
    2.413087844848633
   ],
   "p": [
    0.01842506434482325,
    0.9815749356551768
   ],
   "p_oracle": [
    0.018440761430690376,
    0.9815592385693096
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "no",
   "oracle_selected_option": "no",
   "oracle_margin": 0.9631184771386192,
   "abs_delta_p": [
    1.5697085867124455e-05,
    1.5697085867238947e-05
   ],
   "max_abs_delta_p": 1.5697085867238947e-05,
   "mean_abs_delta_p": 1.56970858671817e-05,
   "finite": true
  },
  {
   "id": "request-09-q03",
   "request_id": "request-09",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-09-q03-option-00",
    "request-09-q03-option-01"
   ],
   "options": [
    "dry",
    "wet"
   ],
   "nopts": 2,
   "scalars": [
    -6.80078125,
    2.177734375
   ],
   "scalars_oracle": [
    -6.80530309677124,
    2.1344544887542725
   ],
   "p": [
    0.005878368209553594,
    0.9941216317904463
   ],
   "p_oracle": [
    0.0060092204092440865,
    0.9939907795907559
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "wet",
   "oracle_selected_option": "wet",
   "oracle_margin": 0.9879815591815118,
   "abs_delta_p": [
    0.00013085219969049232,
    0.00013085219969044548
   ],
   "max_abs_delta_p": 0.00013085219969049232,
   "mean_abs_delta_p": 0.0001308521996904689,
   "finite": true
  },
  {
   "id": "request-10-q01",
   "request_id": "request-10",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-10-q01-option-00",
    "request-10-q01-option-01",
    "request-10-q01-option-02",
    "request-10-q01-option-03",
    "request-10-q01-option-04",
    "request-10-q01-option-05",
    "request-10-q01-option-06",
    "request-10-q01-option-07",
    "request-10-q01-option-08",
    "request-10-q01-option-09",
    "request-10-q01-option-10",
    "request-10-q01-option-11"
   ],
   "options": [
    "rice",
    "sand",
    "water",
    "stones",
    "paper",
    "coins",
    "soil",
    "paint",
    "thread",
    "salt",
    "flour",
    "leaves"
   ],
   "nopts": 12,
   "scalars": [
    0.86083984375,
    -9.265625,
    -9.3515625,
    -11.4375,
    -11.25,
    -11.7578125,
    -11.578125,
    -11.296875,
    -7.4609375,
    -8.7890625,
    -10.1953125,
    -11.4140625
   ],
   "scalars_oracle": [
    0.8400804996490479,
    -9.282438278198242,
    -9.375736236572266,
    -11.433075904846191,
    -11.280050277709961,
    -11.762323379516602,
    -11.585577011108398,
    -11.320062637329102,
    -7.437173843383789,
    -8.777253150939941,
    -10.206853866577148,
    -11.40426254272461
   ],
   "p": [
    0.9749241707382422,
    0.0029915999813037923,
    0.002848239864898017,
    0.0008647947586114116,
    0.0009625972140185835,
    0.0007201478161957803,
    0.0007980211195007364,
    0.000937155616870963,
    0.008390149633684476,
    0.003927997616352906,
    0.0017586709031218782,
    0.0008764547371994746
   ],
   "p_oracle": [
    0.9745978148411945,
    0.0029973496368662185,
    0.002841736544796916,
    0.0008770359084199525,
    0.0009571797033756986,
    0.0007266220906150506,
    0.0008038433781029951,
    0.0009355428472892288,
    0.008603467018522446,
    0.004000445217858127,
    0.0017673671702117771,
    0.0008915956427471904
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "rice",
   "oracle_selected_option": "rice",
   "oracle_margin": 0.965994347822672,
   "abs_delta_p": [
    0.0003263558970477698,
    5.749655562426164e-06,
    6.503320101101159e-06,
    1.2241149808540864e-05,
    5.41751064288493e-06,
    6.474274419270313e-06,
    5.822258602258693e-06,
    1.612769581734179e-06,
    0.00021331738483796985,
    7.244760150522129e-05,
    8.696267089898903e-06,
    1.5140905547715881e-05
   ],
   "max_abs_delta_p": 0.0003263558970477698,
   "mean_abs_delta_p": 5.6648249562232666e-05,
   "finite": true
  },
  {
   "id": "request-10-q02",
   "request_id": "request-10",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-10-q02-option-00",
    "request-10-q02-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    2.732421875,
    -2.494140625
   ],
   "scalars_oracle": [
    2.7351107597351074,
    -2.4982452392578125
   ],
   "p": [
    0.9519654017298504,
    0.04803459827014951
   ],
   "p_oracle": [
    0.9521426038586818,
    0.04785739614131819
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "yes",
   "oracle_selected_option": "yes",
   "oracle_margin": 0.9042852077173635,
   "abs_delta_p": [
    0.00017720212883132191,
    0.00017720212883132191
   ],
   "max_abs_delta_p": 0.00017720212883132191,
   "mean_abs_delta_p": 0.00017720212883132191,
   "finite": true
  },
  {
   "id": "request-10-q03",
   "request_id": "request-10",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-10-q03-option-00",
    "request-10-q03-option-01"
   ],
   "options": [
    "empty",
    "nonempty"
   ],
   "nopts": 2,
   "scalars": [
    -2.60546875,
    2.798828125
   ],
   "scalars_oracle": [
    -2.61518931388855,
    2.7923030853271484
   ],
   "p": [
    0.0435978918440944,
    0.9564021081559057
   ],
   "p_oracle": [
    0.04352181575658243,
    0.9564781842434176
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "nonempty",
   "oracle_selected_option": "nonempty",
   "oracle_margin": 0.9129563684868351,
   "abs_delta_p": [
    7.60760875119712e-05,
    7.607608751192263e-05
   ],
   "max_abs_delta_p": 7.60760875119712e-05,
   "mean_abs_delta_p": 7.607608751194692e-05,
   "finite": true
  },
  {
   "id": "request-11-q01",
   "request_id": "request-11",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-11-q01-option-00",
    "request-11-q01-option-01",
    "request-11-q01-option-02",
    "request-11-q01-option-03",
    "request-11-q01-option-04",
    "request-11-q01-option-05",
    "request-11-q01-option-06",
    "request-11-q01-option-07",
    "request-11-q01-option-08",
    "request-11-q01-option-09",
    "request-11-q01-option-10",
    "request-11-q01-option-11",
    "request-11-q01-option-12",
    "request-11-q01-option-13"
   ],
   "options": [
    "cotton",
    "wool",
    "silk",
    "linen",
    "wood",
    "glass",
    "stone",
    "steel",
    "copper",
    "rubber",
    "paper",
    "clay",
    "leather",
    "plastic"
   ],
   "nopts": 14,
   "scalars": [
    5.23046875,
    -6.1796875,
    -7.18359375,
    -4.40234375,
    -6.56640625,
    -7.4453125,
    -9.1796875,
    -8.3828125,
    -8.640625,
    -9.53125,
    -6.98828125,
    -9.2265625,
    -9.3359375,
    -8.3515625
   ],
   "scalars_oracle": [
    5.246809482574463,
    -6.228903293609619,
    -7.158008575439453,
    -4.403936386108398,
    -6.593619346618652,
    -7.444218635559082,
    -9.191930770874023,
    -8.394148826599121,
    -8.630069732666016,
    -9.541899681091309,
    -7.013782501220703,
    -9.234328269958496,
    -9.323783874511719,
    -8.359192848205566
   ],
   "p": [
    0.9887421573269616,
    0.001456948676936753,
    0.000820930839141845,
    0.0040227642931458035,
    0.0011680809150284572,
    0.0007068972300319654,
    0.00026238525591043994,
    0.0004137123845080587,
    0.0003570406284231762,
    0.0002146311576291547,
    0.0009178610421745992,
    0.00025545037195157143,
    0.00023997341631186735,
    0.00042116646184481565
   ],
   "p_oracle": [
    0.9889120205917192,
    0.001403620839647622,
    0.0008254206927579425,
    0.0039824350139213245,
    0.0011395643482980929,
    0.0007008855319549572,
    0.0002581786875110537,
    0.00040729076030881746,
    0.00035592334436555984,
    0.00021138258655764423,
    0.0008963294960782709,
    0.00025199891705764957,
    0.00023944106269725907,
    0.00041550812712453766
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "cotton",
   "oracle_selected_option": "cotton",
   "oracle_margin": 0.9849295855777979,
   "abs_delta_p": [
    0.00016986326475765257,
    5.332783728913097e-05,
    4.489853616097502e-06,
    4.0329279224479014e-05,
    2.8516566730364375e-05,
    6.0116980770081934e-06,
    4.20656839938624e-06,
    6.42162419924123e-06,
    1.1172840576163736e-06,
    3.2485710715104673e-06,
    2.1531546096328295e-05,
    3.4514548939218654e-06,
    5.32353614608272e-07,
    5.65833472027799e-06
   ],
   "max_abs_delta_p": 0.00016986326475765257,
   "mean_abs_delta_p": 2.4907588339115955e-05,
   "finite": true
  },
  {
   "id": "request-11-q02",
   "request_id": "request-11",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-11-q02-option-00",
    "request-11-q02-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    -3.056640625,
    3.076171875
   ],
   "scalars_oracle": [
    -3.04154896736145,
    3.0489227771759033
   ],
   "p": [
    0.029185474884003165,
    0.9708145251159968
   ],
   "p_oracle": [
    0.029878865074433554,
    0.9701211349255664
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "no",
   "oracle_selected_option": "no",
   "oracle_margin": 0.9402422698511328,
   "abs_delta_p": [
    0.0006933901904303888,
    0.0006933901904303541
   ],
   "max_abs_delta_p": 0.0006933901904303888,
   "mean_abs_delta_p": 0.0006933901904303715,
   "finite": true
  },
  {
   "id": "request-11-q03",
   "request_id": "request-11",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-11-q03-option-00",
    "request-11-q03-option-01",
    "request-11-q03-option-02"
   ],
   "options": [
    "none",
    "some",
    "complete"
   ],
   "nopts": 3,
   "scalars": [
    -1.96875,
    -0.794921875,
    0.211669921875
   ],
   "scalars_oracle": [
    -1.9961133003234863,
    -0.7945132851600647,
    0.13405287265777588
   ],
   "p": [
    0.15547332793355584,
    0.30406245395135767,
    0.5404642181150865
   ],
   "p_oracle": [
    0.15711289848380908,
    0.3121841461234784,
    0.5307029553927125
   ],
   "argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "selected_option": "complete",
   "oracle_selected_option": "complete",
   "oracle_margin": 0.21851880926923412,
   "abs_delta_p": [
    0.0016395705502532387,
    0.00812169217212072,
    0.009761262722373987
   ],
   "max_abs_delta_p": 0.009761262722373987,
   "mean_abs_delta_p": 0.006507508481582648,
   "finite": true
  },
  {
   "id": "request-12-q01",
   "request_id": "request-12",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-12-q01-option-00",
    "request-12-q01-option-01",
    "request-12-q01-option-02",
    "request-12-q01-option-03",
    "request-12-q01-option-04",
    "request-12-q01-option-05",
    "request-12-q01-option-06",
    "request-12-q01-option-07",
    "request-12-q01-option-08",
    "request-12-q01-option-09",
    "request-12-q01-option-10",
    "request-12-q01-option-11",
    "request-12-q01-option-12",
    "request-12-q01-option-13",
    "request-12-q01-option-14",
    "request-12-q01-option-15"
   ],
   "options": [
    "sparrow",
    "robin",
    "raven",
    "owl",
    "duck",
    "goose",
    "swan",
    "finch",
    "gull",
    "hawk",
    "eagle",
    "dove",
    "heron",
    "crane",
    "wren",
    "lark"
   ],
   "nopts": 16,
   "scalars": [
    6.265625,
    -3.2734375,
    -6.76953125,
    -7.04296875,
    -7.17578125,
    -7.4609375,
    -7.1640625,
    -2.654296875,
    -6.59375,
    -7.6796875,
    -8.28125,
    -4.83203125,
    -8.1015625,
    -7.0078125,
    -2.908203125,
    -3.25390625
   ],
   "scalars_oracle": [
    6.2408976554870605,
    -3.3339803218841553,
    -6.770749568939209,
    -7.012792587280273,
    -7.17970609664917,
    -7.4471611976623535,
    -7.164470195770264,
    -2.6470611095428467,
    -6.595610618591309,
    -7.668771743774414,
    -8.25868034362793,
    -4.801041126251221,
    -8.084135055541992,
    -7.016900062561035,
    -2.95692777633667,
    -3.219796895980835
   ],
   "p": [
    0.974457554128196,
    0.004182830223138293,
    0.0005673495057783591,
    0.0004852797487176954,
    0.0004498133275299724,
    0.00038217791276927,
    0.00045283557819743563,
    0.005958285352481112,
    0.0006272981554897692,
    0.0003372708242551264,
    0.00023916082663506745,
    0.0017166300647784584,
    0.00026502252220973456,
    0.000495127255746828,
    0.005153589496149393,
    0.004229775077927515
   ],
   "p_oracle": [
    0.9742324389470528,
    0.004097147806356085,
    0.0005748897092791492,
    0.0005006303058684603,
    0.0004550870721678767,
    0.00039058955678796583,
    0.00045906646207074066,
    0.006066709336247993,
    0.0006354018368255835,
    0.0003441310981245799,
    0.000245655968598753,
    0.001771755060649297,
    0.00027142129832778033,
    0.000499456640004762,
    0.005082227613647842,
    0.00437339128799038
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "sparrow",
   "oracle_selected_option": "sparrow",
   "oracle_margin": 0.9681657296108048,
   "abs_delta_p": [
    0.0002251151811432095,
    8.568241678220782e-05,
    7.540203500790111e-06,
    1.535055715076489e-05,
    5.273744637904318e-06,
    8.411644018695846e-06,
    6.230883873305031e-06,
    0.00010842398376688118,
    8.103681335814347e-06,
    6.860273869453454e-06,
    6.4951419636855696e-06,
    5.512499587083863e-05,
    6.398776118045769e-06,
    4.32938425793403e-06,
    7.136188250155169e-05,
    0.00014361621006286473
   ],
   "max_abs_delta_p": 0.0002251151811432095,
   "mean_abs_delta_p": 4.776993505337168e-05,
   "finite": true
  },
  {
   "id": "request-12-q02",
   "request_id": "request-12",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-12-q02-option-00",
    "request-12-q02-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    3.04296875,
    -2.228515625
   ],
   "scalars_oracle": [
    3.03344464302063,
    -2.245439291000366
   ],
   "p": [
    0.9531256795529978,
    0.046874320447002335
   ],
   "p_oracle": [
    0.9533142271512312,
    0.04668577284876895
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "yes",
   "oracle_selected_option": "yes",
   "oracle_margin": 0.9066284543024622,
   "abs_delta_p": [
    0.00018854759823339062,
    0.00018854759823338368
   ],
   "max_abs_delta_p": 0.00018854759823339062,
   "mean_abs_delta_p": 0.00018854759823338715,
   "finite": true
  },
  {
   "id": "request-12-q03",
   "request_id": "request-12",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-12-q03-option-00",
    "request-12-q03-option-01",
    "request-12-q03-option-02"
   ],
   "options": [
    "still",
    "slightly active",
    "active"
   ],
   "nopts": 3,
   "scalars": [
    -2.693359375,
    -1.3134765625,
    1.490234375
   ],
   "scalars_oracle": [
    -2.706667423248291,
    -1.3185935020446777,
    1.510910987854004
   ],
   "p": [
    0.07081942000939169,
    0.15581006311278342,
    0.7733705168778249
   ],
   "p_oracle": [
    0.06971125616279088,
    0.15409154856746585,
    0.7761971952697433
   ],
   "argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "selected_option": "active",
   "oracle_selected_option": "active",
   "oracle_margin": 0.6221056467022774,
   "abs_delta_p": [
    0.0011081638466008126,
    0.0017185145453175732,
    0.0028266783919184135
   ],
   "max_abs_delta_p": 0.0028266783919184135,
   "mean_abs_delta_p": 0.0018844522612789332,
   "finite": true
  },
  {
   "id": "request-13-q01",
   "request_id": "request-13",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-13-q01-option-00",
    "request-13-q01-option-01"
   ],
   "options": [
    "send an intact replacement",
    "cancel all future deliveries"
   ],
   "nopts": 2,
   "scalars": [
    4.39453125,
    -9.6171875
   ],
   "scalars_oracle": [
    4.405121803283691,
    -9.618844985961914
   ],
   "p": [
    0.9996668872700176,
    0.00033311272998246975
   ],
   "p_oracle": [
    0.9996692097777317,
    0.0003307902222684505
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "send an intact replacement",
   "oracle_selected_option": "send an intact replacement",
   "oracle_margin": 0.9993384195554632,
   "abs_delta_p": [
    2.3225077140365258e-06,
    2.3225077140192328e-06
   ],
   "max_abs_delta_p": 2.3225077140365258e-06,
   "mean_abs_delta_p": 2.3225077140278793e-06,
   "finite": true
  },
  {
   "id": "request-13-q02",
   "request_id": "request-13",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-13-q02-option-00",
    "request-13-q02-option-01"
   ],
   "options": [
    "the bowl was damaged",
    "the address was missing"
   ],
   "nopts": 2,
   "scalars": [
    5.375,
    -10.1875
   ],
   "scalars_oracle": [
    5.3750176429748535,
    -10.238785743713379
   ],
   "p": [
    0.9998626522462887,
    0.00013734775371118408
   ],
   "p_oracle": [
    0.9998666197855478,
    0.00013338021445225527
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "the bowl was damaged",
   "oracle_selected_option": "the bowl was damaged",
   "oracle_margin": 0.9997332395710955,
   "abs_delta_p": [
    3.967539259130959e-06,
    3.967539258928809e-06
   ],
   "max_abs_delta_p": 3.967539259130959e-06,
   "mean_abs_delta_p": 3.967539259029884e-06,
   "finite": true
  },
  {
   "id": "request-13-q03",
   "request_id": "request-13",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-13-q03-option-00",
    "request-13-q03-option-01"
   ],
   "options": [
    "a complaint about damage",
    "a request for directions"
   ],
   "nopts": 2,
   "scalars": [
    7.0546875,
    -10.8203125
   ],
   "scalars_oracle": [
    7.067084312438965,
    -10.796977996826172
   ],
   "p": [
    0.9999633582536426,
    3.664174635732589e-05
   ],
   "p_oracle": [
    0.9999631285300276,
    3.687146997256911e-05
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "a complaint about damage",
   "oracle_selected_option": "a complaint about damage",
   "oracle_margin": 0.999926257060055,
   "abs_delta_p": [
    2.297236150194948e-07,
    2.297236152432199e-07
   ],
   "max_abs_delta_p": 2.297236152432199e-07,
   "mean_abs_delta_p": 2.2972361513135735e-07,
   "finite": true
  },
  {
   "id": "request-13-q04",
   "request_id": "request-13",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-13-q04-option-00",
    "request-13-q04-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    4.59375,
    -1.1025390625
   ],
   "scalars_oracle": [
    4.599853515625,
    -1.0872465372085571
   ],
   "p": [
    0.9628531639366981,
    0.03714683606330199
   ],
   "p_oracle": [
    0.9626648994801141,
    0.037335100519885804
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "yes",
   "oracle_selected_option": "yes",
   "oracle_margin": 0.9253297989602284,
   "abs_delta_p": [
    0.00018826445658393087,
    0.0001882644565838129
   ],
   "max_abs_delta_p": 0.00018826445658393087,
   "mean_abs_delta_p": 0.0001882644565838719,
   "finite": true
  },
  {
   "id": "request-13-q05",
   "request_id": "request-13",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-13-q05-option-00",
    "request-13-q05-option-01",
    "request-13-q05-option-02",
    "request-13-q05-option-03"
   ],
   "options": [
    "unusable",
    "limited",
    "mostly usable",
    "fully usable"
   ],
   "nopts": 4,
   "scalars": [
    1.6171875,
    0.30712890625,
    -1.2392578125,
    -3.888671875
   ],
   "scalars_oracle": [
    1.6196714639663696,
    0.27900874614715576,
    -1.3120677471160889,
    -3.97291898727417
   ],
   "p": [
    0.5842727542749888,
    0.2763762074563208,
    0.11421851986295291,
    0.02513251840573747
   ],
   "p_oracle": [
    0.5906616660481768,
    0.274554667682575,
    0.11060483754235892,
    0.024178828726889312
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "unusable",
   "oracle_selected_option": "unusable",
   "oracle_margin": 0.3161069983656018,
   "abs_delta_p": [
    0.006388911773188011,
    0.0018215397737457728,
    0.003613682320593989,
    0.0009536896788481589
   ],
   "max_abs_delta_p": 0.006388911773188011,
   "mean_abs_delta_p": 0.003194455886593983,
   "finite": true
  },
  {
   "id": "request-14-q01",
   "request_id": "request-14",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-14-q01-option-00",
    "request-14-q01-option-01"
   ],
   "options": [
    "a pleasant quiet stay",
    "a noisy delayed arrival"
   ],
   "nopts": 2,
   "scalars": [
    3.591796875,
    -10.9453125
   ],
   "scalars_oracle": [
    3.573038101196289,
    -10.914566040039062
   ],
   "p": [
    0.9997532577813535,
    0.0002467422186465334
   ],
   "p_oracle": [
    0.999746179895068,
    0.000253820104931945
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "a pleasant quiet stay",
   "oracle_selected_option": "a pleasant quiet stay",
   "oracle_margin": 0.9994923597901361,
   "abs_delta_p": [
    7.0778862855114255e-06,
    7.077886285411625e-06
   ],
   "max_abs_delta_p": 7.0778862855114255e-06,
   "mean_abs_delta_p": 7.077886285461525e-06,
   "finite": true
  },
  {
   "id": "request-14-q02",
   "request_id": "request-14",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-14-q02-option-00",
    "request-14-q02-option-01"
   ],
   "options": [
    "clean and ready early",
    "dirty and still locked"
   ],
   "nopts": 2,
   "scalars": [
    -0.489013671875,
    -9.7109375
   ],
   "scalars_oracle": [
    -0.42073899507522583,
    -9.746484756469727
   ],
   "p": [
    0.9948810342967479,
    0.005118965703252212
   ],
   "p_oracle": [
    0.995174470397323,
    0.004825529602676933
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "clean and ready early",
   "oracle_selected_option": "clean and ready early",
   "oracle_margin": 0.9903489407946461,
   "abs_delta_p": [
    0.00029343610057508407,
    0.0002934361005752792
   ],
   "max_abs_delta_p": 0.0002934361005752792,
   "mean_abs_delta_p": 0.00029343610057518165,
   "finite": true
  },
  {
   "id": "request-14-q03",
   "request_id": "request-14",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-14-q03-option-00",
    "request-14-q03-option-01"
   ],
   "options": [
    "pleased with the experience",
    "angry about the delay"
   ],
   "nopts": 2,
   "scalars": [
    2.412109375,
    -10.5234375
   ],
   "scalars_oracle": [
    2.417165756225586,
    -10.482742309570312
   ],
   "p": [
    0.9993840576605115,
    0.0006159423394884628
   ],
   "p_oracle": [
    0.9993713933420288,
    0.0006286066579710768
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "pleased with the experience",
   "oracle_selected_option": "pleased with the experience",
   "oracle_margin": 0.9987427866840578,
   "abs_delta_p": [
    1.2664318482680237e-05,
    1.2664318482613992e-05
   ],
   "max_abs_delta_p": 1.2664318482680237e-05,
   "mean_abs_delta_p": 1.2664318482647114e-05,
   "finite": true
  },
  {
   "id": "request-14-q04",
   "request_id": "request-14",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-14-q04-option-00",
    "request-14-q04-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    -1.6494140625,
    1.400390625
   ],
   "scalars_oracle": [
    -1.5531220436096191,
    1.3508743047714233
   ],
   "p": [
    0.14896453614240898,
    0.851035463857591
   ],
   "p_oracle": [
    0.15983899956016281,
    0.8401610004398371
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "no",
   "oracle_selected_option": "no",
   "oracle_margin": 0.6803220008796743,
   "abs_delta_p": [
    0.010874463417753832,
    0.010874463417753888
   ],
   "max_abs_delta_p": 0.010874463417753888,
   "mean_abs_delta_p": 0.01087446341775386,
   "finite": true
  },
  {
   "id": "request-14-q05",
   "request_id": "request-14",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-14-q05-option-00",
    "request-14-q05-option-01",
    "request-14-q05-option-02",
    "request-14-q05-option-03"
   ],
   "options": [
    "poor",
    "fair",
    "good",
    "excellent"
   ],
   "nopts": 4,
   "scalars": [
    -7.9453125,
    -3.16015625,
    -0.0023174285888671875,
    -1.28515625
   ],
   "scalars_oracle": [
    -7.972799301147461,
    -3.1746554374694824,
    -0.007742464542388916,
    -1.2937160730361938
   ],
   "p": [
    0.006453902866914277,
    0.0993908486428359,
    0.6039789642800383,
    0.2901762842102117
   ],
   "p_oracle": [
    0.00638015986060032,
    0.09898711073805981,
    0.6046526754736612,
    0.28998005392767867
   ],
   "argmax": 2,
   "oracle_argmax": 2,
   "argmax_agrees": true,
   "selected_option": "good",
   "oracle_selected_option": "good",
   "oracle_margin": 0.3146726215459825,
   "abs_delta_p": [
    7.374300631395742e-05,
    0.0004037379047760886,
    0.0006737111936229168,
    0.00019623028253301644
   ],
   "max_abs_delta_p": 0.0006737111936229168,
   "mean_abs_delta_p": 0.0003368555968114948,
   "finite": true
  },
  {
   "id": "request-15-q01",
   "request_id": "request-15",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-15-q01-option-00",
    "request-15-q01-option-01"
   ],
   "options": [
    "arrange an urgent repair",
    "update the wall color"
   ],
   "nopts": 2,
   "scalars": [
    -0.61669921875,
    -14.59375
   ],
   "scalars_oracle": [
    -0.6910357475280762,
    -14.593600273132324
   ],
   "p": [
    0.9996602246838883,
    0.00033977531611177375
   ],
   "p_oracle": [
    0.9996454556780844,
    0.00035454432191554705
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "arrange an urgent repair",
   "oracle_selected_option": "arrange an urgent repair",
   "oracle_margin": 0.9992909113561689,
   "abs_delta_p": [
    1.4769005803882962e-05,
    1.4769005803773295e-05
   ],
   "max_abs_delta_p": 1.4769005803882962e-05,
   "mean_abs_delta_p": 1.4769005803828128e-05,
   "finite": true
  },
  {
   "id": "request-15-q02",
   "request_id": "request-15",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-15-q02-option-00",
    "request-15-q02-option-01"
   ],
   "options": [
    "an alternative access route",
    "a printed lunch menu"
   ],
   "nopts": 2,
   "scalars": [
    6.65625,
    -14.5703125
   ],
   "scalars_oracle": [
    6.661491870880127,
    -14.577804565429688
   ],
   "p": [
    0.9999946019332825,
    5.398066717386745e-06
   ],
   "p_oracle": [
    0.9999946410697321,
    5.358930267851325e-06
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "an alternative access route",
   "oracle_selected_option": "an alternative access route",
   "oracle_margin": 0.9999892821394643,
   "abs_delta_p": [
    3.91364496366009e-08,
    3.9136449535420274e-08
   ],
   "max_abs_delta_p": 3.91364496366009e-08,
   "mean_abs_delta_p": 3.913644958601059e-08,
   "finite": true
  },
  {
   "id": "request-15-q03",
   "request_id": "request-15",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-15-q03-option-00",
    "request-15-q03-option-01"
   ],
   "options": [
    "building access and maintenance",
    "gardening plans and supplies"
   ],
   "nopts": 2,
   "scalars": [
    -1.990234375,
    -10.9296875
   ],
   "scalars_oracle": [
    -2.031851053237915,
    -11.027084350585938
   ],
   "p": [
    0.9939897403150901,
    0.006010259684909857
   ],
   "p_oracle": [
    0.99417719477895,
    0.005822805221049984
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "building access and maintenance",
   "oracle_selected_option": "building access and maintenance",
   "oracle_margin": 0.9883543895579,
   "abs_delta_p": [
    0.00018745446385981523,
    0.00018745446385987248
   ],
   "max_abs_delta_p": 0.00018745446385987248,
   "mean_abs_delta_p": 0.00018745446385984386,
   "finite": true
  },
  {
   "id": "request-15-q04",
   "request_id": "request-15",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-15-q04-option-00",
    "request-15-q04-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    -1.669921875,
    1.3291015625
   ],
   "scalars_oracle": [
    -1.7337946891784668,
    1.3656667470932007
   ],
   "p": [
    0.1526808435988674,
    0.8473191564011325
   ],
   "p_oracle": [
    0.14540299930898767,
    0.8545970006910122
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "no",
   "oracle_selected_option": "no",
   "oracle_margin": 0.7091940013820245,
   "abs_delta_p": [
    0.007277844289879737,
    0.007277844289879654
   ],
   "max_abs_delta_p": 0.007277844289879737,
   "mean_abs_delta_p": 0.007277844289879695,
   "finite": true
  },
  {
   "id": "request-15-q05",
   "request_id": "request-15",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-15-q05-option-00",
    "request-15-q05-option-01",
    "request-15-q05-option-02",
    "request-15-q05-option-03",
    "request-15-q05-option-04"
   ],
   "options": [
    "very low",
    "low",
    "medium",
    "high",
    "critical"
   ],
   "nopts": 5,
   "scalars": [
    -6.43359375,
    -2.796875,
    -0.83154296875,
    1.4765625,
    -0.794921875
   ],
   "scalars_oracle": [
    -6.44263219833374,
    -2.8297576904296875,
    -0.8664757013320923,
    1.476930022239685,
    -0.7765030860900879
   ],
   "p": [
    0.006645610787528814,
    0.05309493856616944,
    0.16322481436880665,
    0.6103581344320044,
    0.1666765018454907
   ],
   "p_oracle": [
    0.006626994094001005,
    0.052229686529919314,
    0.16037686679369648,
    0.61192850142172,
    0.16883795116066314
   ],
   "argmax": 3,
   "oracle_argmax": 3,
   "argmax_agrees": true,
   "selected_option": "high",
   "oracle_selected_option": "high",
   "oracle_margin": 0.4430905502610568,
   "abs_delta_p": [
    1.861669352780891e-05,
    0.0008652520362501248,
    0.0028479475751101713,
    0.001570366989715577,
    0.0021614493151724257
   ],
   "max_abs_delta_p": 0.0028479475751101713,
   "mean_abs_delta_p": 0.0014927265219552214,
   "finite": true
  },
  {
   "id": "request-16-q01",
   "request_id": "request-16",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-16-q01-option-00",
    "request-16-q01-option-01"
   ],
   "options": [
    "a minor visible blemish",
    "a completely unreadable page"
   ],
   "nopts": 2,
   "scalars": [
    3.568359375,
    -8.1953125
   ],
   "scalars_oracle": [
    3.515467405319214,
    -8.229701042175293
   ],
   "p": [
    0.9987974386028178,
    0.001202561397182106
   ],
   "p_oracle": [
    0.9987846715320305,
    0.0012153284679694153
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "a minor visible blemish",
   "oracle_selected_option": "a minor visible blemish",
   "oracle_margin": 0.9975693430640611,
   "abs_delta_p": [
    1.2767070787234047e-05,
    1.276707078730929e-05
   ],
   "max_abs_delta_p": 1.276707078730929e-05,
   "mean_abs_delta_p": 1.2767070787271669e-05,
   "finite": true
  },
  {
   "id": "request-16-q02",
   "request_id": "request-16",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-16-q02-option-00",
    "request-16-q02-option-01"
   ],
   "options": [
    "all of the text",
    "none of the text"
   ],
   "nopts": 2,
   "scalars": [
    3.326171875,
    -5.1171875
   ],
   "scalars_oracle": [
    3.330043077468872,
    -5.142807960510254
   ],
   "p": [
    0.9920355952826015,
    0.007964404717398515
   ],
   "p_oracle": [
    0.992167647375149,
    0.007832352624851022
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "all of the text",
   "oracle_selected_option": "all of the text",
   "oracle_margin": 0.9843352947502979,
   "abs_delta_p": [
    0.0001320520925474078,
    0.0001320520925474928
   ],
   "max_abs_delta_p": 0.0001320520925474928,
   "mean_abs_delta_p": 0.0001320520925474503,
   "finite": true
  },
  {
   "id": "request-16-q03",
   "request_id": "request-16",
   "type": "choice",
   "zoo_only": false,
   "row_ids": [
    "request-16-q03-option-00",
    "request-16-q03-option-01"
   ],
   "options": [
    "keep the readable copy",
    "report a missing parcel"
   ],
   "nopts": 2,
   "scalars": [
    -0.65087890625,
    -13.2109375
   ],
   "scalars_oracle": [
    -0.6904910802841187,
    -13.2025728225708
   ],
   "p": [
    0.9992367614051184,
    0.0007632385948815458
   ],
   "p_oracle": [
    0.9992155641329983,
    0.0007844358670016086
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "keep the readable copy",
   "oracle_selected_option": "keep the readable copy",
   "oracle_margin": 0.9984311282659967,
   "abs_delta_p": [
    2.119727212013256e-05,
    2.1197272120062846e-05
   ],
   "max_abs_delta_p": 2.119727212013256e-05,
   "mean_abs_delta_p": 2.1197272120097703e-05,
   "finite": true
  },
  {
   "id": "request-16-q04",
   "request_id": "request-16",
   "type": "noul",
   "zoo_only": false,
   "row_ids": [
    "request-16-q04-option-00",
    "request-16-q04-option-01"
   ],
   "options": [
    "yes",
    "no"
   ],
   "nopts": 2,
   "scalars": [
    2.197265625,
    -2.68359375
   ],
   "scalars_oracle": [
    2.1954517364501953,
    -2.713536024093628
   ],
   "p": [
    0.9420819125605256,
    0.05791808743947446
   ],
   "p_oracle": [
    0.942952726312319,
    0.057047273687681144
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "yes",
   "oracle_selected_option": "yes",
   "oracle_margin": 0.8859054526246378,
   "abs_delta_p": [
    0.0008708137517933201,
    0.0008708137517933132
   ],
   "max_abs_delta_p": 0.0008708137517933201,
   "mean_abs_delta_p": 0.0008708137517933166,
   "finite": true
  },
  {
   "id": "request-16-q05",
   "request_id": "request-16",
   "type": "score",
   "zoo_only": false,
   "row_ids": [
    "request-16-q05-option-00",
    "request-16-q05-option-01",
    "request-16-q05-option-02",
    "request-16-q05-option-03",
    "request-16-q05-option-04"
   ],
   "options": [
    "very low",
    "low",
    "medium",
    "high",
    "critical"
   ],
   "nopts": 5,
   "scalars": [
    -2.490234375,
    -0.1806640625,
    -1.2763671875,
    -3.6640625,
    -7.6796875
   ],
   "scalars_oracle": [
    -2.494438886642456,
    -0.20091819763183594,
    -1.2986819744110107,
    -3.687429666519165,
    -7.6829833984375
   ],
   "p": [
    0.1368673365738588,
    0.5122263282801905,
    0.2738691680153992,
    0.06998305784226831,
    0.007054109288283194
   ],
   "p_oracle": [
    0.13800648634764923,
    0.5117744176558293,
    0.27330553650463313,
    0.0697970447117941,
    0.007116514780093998
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "low",
   "oracle_selected_option": "low",
   "oracle_margin": 0.2384688811511962,
   "abs_delta_p": [
    0.0011391497737904266,
    0.0004519106243612292,
    0.0005636315107660494,
    0.00018601313047421353,
    6.240549181080451e-05
   ],
   "max_abs_delta_p": 0.0011391497737904266,
   "mean_abs_delta_p": 0.0004806221062405447,
   "finite": true
  },
  {
   "id": "request-17-q01",
   "request_id": "request-17",
   "type": "choice",
   "zoo_only": true,
   "row_ids": [
    "request-17-q01-option-00",
    "request-17-q01-option-01",
    "request-17-q01-option-02",
    "request-17-q01-option-03",
    "request-17-q01-option-04",
    "request-17-q01-option-05",
    "request-17-q01-option-06",
    "request-17-q01-option-07",
    "request-17-q01-option-08",
    "request-17-q01-option-09",
    "request-17-q01-option-10",
    "request-17-q01-option-11",
    "request-17-q01-option-12",
    "request-17-q01-option-13",
    "request-17-q01-option-14",
    "request-17-q01-option-15",
    "request-17-q01-option-16",
    "request-17-q01-option-17",
    "request-17-q01-option-18",
    "request-17-q01-option-19"
   ],
   "options": [
    "lighting",
    "plumbing",
    "heating",
    "cooling",
    "doors",
    "windows",
    "floors",
    "walls",
    "ceilings",
    "roofs",
    "stairs",
    "lifts",
    "locks",
    "shelves",
    "desks",
    "chairs",
    "curtains",
    "carpets",
    "drains",
    "pipes"
   ],
   "nopts": 20,
   "scalars": [
    5.546875,
    -7.29296875,
    -7.6171875,
    -7.91796875,
    -8.34375,
    -8.515625,
    -8.515625,
    -6.609375,
    2.87109375,
    -6.01953125,
    -8.578125,
    -8.8828125,
    -8.84375,
    -9.9296875,
    -9.8984375,
    -9.734375,
    -10.65625,
    -10.4453125,
    -9.1640625,
    -7.57421875
   ],
   "scalars_oracle": [
    5.513814926147461,
    -7.31196403503418,
    -7.631340503692627,
    -7.937613487243652,
    -8.349449157714844,
    -8.52861499786377,
    -8.547520637512207,
    -6.649753093719482,
    2.8335750102996826,
    -5.990054130554199,
    -8.59441089630127,
    -8.899070739746094,
    -8.87210464477539,
    -9.984298706054688,
    -9.920188903808594,
    -9.742130279541016,
    -10.690881729125977,
    -10.486421585083008,
    -9.18571662902832,
    -7.604816436767578
   ],
   "p": [
    0.8170027653991307,
    0.0005318407951026011,
    0.00044189678207370923,
    0.0003721143985890435,
    0.0002917501422255933,
    0.00026445827091722243,
    0.00026445827091722243,
    0.0007860097423732489,
    0.1770840754844727,
    0.001101053534052097,
    0.0002551800022291761,
    0.0002144039911307767,
    0.00021924360713401834,
    0.00011787771324311435,
    0.000120001579058345,
    0.0001317959510932551,
    7.78255035995484e-05,
    8.779503073736125e-05,
    0.0001825725738046415,
    0.0004528812281155837
   ],
   "p_oracle": [
    0.8173185076199884,
    0.0005363396301957823,
    0.00044686956878130105,
    0.00037512287018886067,
    0.0002964619730113619,
    0.00026761208213463506,
    0.0002647365695935582,
    0.0007830322250443364,
    0.17670173583366977,
    0.0011415527626462557,
    0.00025773729274020525,
    0.00021655606655843212,
    0.000219918864143094,
    0.00011647995701225297,
    0.00012082622884148439,
    0.00013376722645384417,
    7.778569053030336e-05,
    8.742592434916856e-05,
    0.0001838374293344633,
    0.00045369418478281734
   ],
   "argmax": 0,
   "oracle_argmax": 0,
   "argmax_agrees": true,
   "selected_option": "lighting",
   "oracle_selected_option": "lighting",
   "oracle_margin": 0.6406167717863186,
   "abs_delta_p": [
    0.0003157422208576177,
    4.498835093181262e-06,
    4.972786707591818e-06,
    3.008471599817177e-06,
    4.711830785768592e-06,
    3.1538112174126225e-06,
    2.782986763357658e-07,
    2.9775173289124917e-06,
    0.00038233965080292087,
    4.049922859415862e-05,
    2.5572905110291693e-06,
    2.1520754276554163e-06,
    6.752570090756726e-07,
    1.3977562308613794e-06,
    8.246497831393949e-07,
    1.9712753605890653e-06,
    3.981306924503297e-08,
    3.6910638819268504e-07,
    1.2648555298217976e-06,
    8.129566672336499e-07
   ],
   "max_abs_delta_p": 0.00038233965080292087,
   "mean_abs_delta_p": 3.871238438202801e-05,
   "finite": true
  },
  {
   "id": "request-18-q01",
   "request_id": "request-18",
   "type": "choice",
   "zoo_only": true,
   "row_ids": [
    "request-18-q01-option-00",
    "request-18-q01-option-01",
    "request-18-q01-option-02",
    "request-18-q01-option-03",
    "request-18-q01-option-04",
    "request-18-q01-option-05",
    "request-18-q01-option-06",
    "request-18-q01-option-07",
    "request-18-q01-option-08",
    "request-18-q01-option-09",
    "request-18-q01-option-10",
    "request-18-q01-option-11",
    "request-18-q01-option-12",
    "request-18-q01-option-13",
    "request-18-q01-option-14",
    "request-18-q01-option-15",
    "request-18-q01-option-16",
    "request-18-q01-option-17",
    "request-18-q01-option-18",
    "request-18-q01-option-19",
    "request-18-q01-option-20",
    "request-18-q01-option-21",
    "request-18-q01-option-22",
    "request-18-q01-option-23"
   ],
   "options": [
    "lighting",
    "plumbing",
    "heating",
    "cooling",
    "doors",
    "windows",
    "floors",
    "walls",
    "ceilings",
    "roofs",
    "stairs",
    "lifts",
    "locks",
    "shelves",
    "desks",
    "chairs",
    "curtains",
    "carpets",
    "drains",
    "pipes",
    "valves",
    "tanks",
    "pumps",
    "fans"
   ],
   "nopts": 24,
   "scalars": [
    -8.0703125,
    3.806640625,
    -6.81640625,
    -8.1328125,
    -8.90625,
    -8.390625,
    -6.48046875,
    -5.98828125,
    -5.34765625,
    -6.62109375,
    -9.8671875,
    -9.421875,
    -9.2421875,
    -9.8046875,
    -9.6953125,
    -10.8203125,
    -11.6953125,
    -10.6484375,
    -3.96484375,
    1.8759765625,
    -5.203125,
    -6.12109375,
    -6.74609375,
    -8.1484375
   ],
   "scalars_oracle": [
    -8.074076652526855,
    3.807345151901245,
    -6.854000091552734,
    -8.165847778320312,
    -8.922479629516602,
    -8.393821716308594,
    -6.5011138916015625,
    -6.00640869140625,
    -5.335151672363281,
    -6.5979790687561035,
    -9.877313613891602,
    -9.465591430664062,
    -9.240874290466309,
    -9.860201835632324,
    -9.681700706481934,
    -10.806360244750977,
    -11.715137481689453,
    -10.642301559448242,
    -3.9700522422790527,
    1.8557240962982178,
    -5.19008207321167,
    -6.140894889831543,
    -6.777218341827393,
    -8.173320770263672
   ],
   "p": [
    0.0008178288868364021,
    0.7246785723650602,
    0.0016743343914870587,
    0.0007891361326767512,
    0.0005072350768177394,
    0.0006810375305959789,
    0.0020286691639412858,
    0.0026875499888926457,
    0.0038756036153229922,
    0.0018720289654675123,
    0.000292910591984073,
    0.00037778743695541253,
    0.0004186395439828854,
    0.0003035607336498069,
    0.0003231387189152597,
    0.00016990246863393167,
    0.00010305105638734358,
    0.00018743626061120422,
    0.008541016200882906,
    0.24044505476816364,
    0.004209276196470666,
    0.002491131778322364,
    0.0017429764922746284,
    0.0007821216356672945
   ],
   "p_oracle": [
    0.0008182089163662708,
    0.7268690316279388,
    0.0016430413896767416,
    0.0007764071494108794,
    0.0005038688428868821,
    0.0006815749601400866,
    0.0020101277703289343,
    0.0026668205916226774,
    0.0039136185745641825,
    0.001901887392480624,
    0.0002919832922165861,
    0.00036943189294090403,
    0.0004200510265068098,
    0.00029485235549343234,
    0.00032651483575018203,
    0.0001717109934837397,
    0.00010215706837264009,
    0.00018858722148271857,
    0.008537935465879583,
    0.23830092850425097,
    0.004251871872072085,
    0.0024695543690186777,
    0.0017167351455453683,
    0.000773098741569937
   ],
   "argmax": 1,
   "oracle_argmax": 1,
   "argmax_agrees": true,
   "selected_option": "plumbing",
   "oracle_selected_option": "plumbing",
   "oracle_margin": 0.4885681031236878,
   "abs_delta_p": [
    3.8002952986869627e-07,
    0.0021904592628786412,
    3.129300181031703e-05,
    1.2728983265871763e-05,
    3.366233930857305e-06,
    5.374295441076381e-07,
    1.8541393612351437e-05,
    2.072939726996837e-05,
    3.801495924119023e-05,
    2.985842701311173e-05,
    9.272997674868882e-07,
    8.3555440145085e-06,
    1.4114825239243986e-06,
    8.708378156374553e-06,
    3.376116834922355e-06,
    1.8085248498080173e-06,
    8.939880147034915e-07,
    1.1509608715143545e-06,
    3.0807350033228847e-06,
    0.0021441262639126657,
    4.2595675601418866e-05,
    2.1577409303686484e-05,
    2.6241346729260085e-05,
    9.02289409735748e-06
   ],
   "max_abs_delta_p": 0.0021904592628786412,
   "mean_abs_delta_p": 0.000192466072407385,
   "finite": true
  },
  {
   "id": "request-19-q01",
   "request_id": "request-19",
   "type": "choice",
   "zoo_only": true,
   "row_ids": [
    "request-19-q01-option-00",
    "request-19-q01-option-01",
    "request-19-q01-option-02",
    "request-19-q01-option-03",
    "request-19-q01-option-04",
    "request-19-q01-option-05",
    "request-19-q01-option-06",
    "request-19-q01-option-07",
    "request-19-q01-option-08",
    "request-19-q01-option-09",
    "request-19-q01-option-10",
    "request-19-q01-option-11",
    "request-19-q01-option-12",
    "request-19-q01-option-13",
    "request-19-q01-option-14",
    "request-19-q01-option-15",
    "request-19-q01-option-16",
    "request-19-q01-option-17",
    "request-19-q01-option-18",
    "request-19-q01-option-19",
    "request-19-q01-option-20",
    "request-19-q01-option-21",
    "request-19-q01-option-22",
    "request-19-q01-option-23",
    "request-19-q01-option-24",
    "request-19-q01-option-25",
    "request-19-q01-option-26",
    "request-19-q01-option-27"
   ],
   "options": [
    "lighting",
    "plumbing",
    "heating",
    "cooling",
    "doors",
    "windows",
    "floors",
    "walls",
    "ceilings",
    "roofs",
    "stairs",
    "lifts",
    "locks",
    "shelves",
    "desks",
    "chairs",
    "curtains",
    "carpets",
    "drains",
    "pipes",
    "valves",
    "tanks",
    "pumps",
    "fans",
    "sockets",
    "switches",
    "cables",
    "sensors"
   ],
   "nopts": 28,
   "scalars": [
    -7.59765625,
    -8.25,
    -7.59765625,
    -0.9609375,
    -9.203125,
    -8.6796875,
    -10.09375,
    -9.671875,
    -5.91015625,
    -9.5859375,
    -10.515625,
    -10.109375,
    -8.046875,
    -9.9140625,
    -10.5859375,
    -9.6953125,
    -11.84375,
    -11.140625,
    -9.34375,
    -8.171875,
    -8.9296875,
    -7.9453125,
    -6.96484375,
    1.26953125,
    -7.0859375,
    -7.0234375,
    -7.6796875,
    -7.4765625
   ],
   "scalars_oracle": [
    -7.635459899902344,
    -8.273568153381348,
    -7.611283779144287,
    -1.0317230224609375,
    -9.20922565460205,
    -8.708393096923828,
    -10.114972114562988,
    -9.697352409362793,
    -5.983921051025391,
    -9.618802070617676,
    -10.518702507019043,
    -10.130147933959961,
    -8.015636444091797,
    -9.947691917419434,
    -10.613463401794434,
    -9.693241119384766,
    -11.859024047851562,
    -11.160334587097168,
    -9.379319190979004,
    -8.208106994628906,
    -8.927600860595703,
    -7.999220848083496,
    -6.959987640380859,
    1.2542405128479004,
    -7.119679927825928,
    -7.052636623382568,
    -7.682580471038818,
    -7.514122486114502
   ],
   "p": [
    0.004526783387353771,
    0.0031181621105346538,
    0.004526783387353771,
    0.20082268058215394,
    0.0018086865230514972,
    0.0024392925209502617,
    0.0010872725715984505,
    0.0013836760489464956,
    0.011873361156765551,
    0.0014533204499990575,
    0.0008543630178829675,
    0.0010776079904366784,
    0.003501933307932868,
    0.0012048449710962612,
    0.0008207163951989388,
    0.0013652682151830736,
    0.00039998522604067495,
    0.0005977743405616494,
    0.0016690319056385683,
    0.00326051976002208,
    0.0021145687774376896,
    0.0037111837803048627,
    0.006498807906726891,
    0.7183636405308844,
    0.006064319485437097,
    0.006284816331210735,
    0.00431948688361155,
    0.004851112435685633
   ],
   "p_oracle": [
    0.004499781253382536,
    0.003124878797177552,
    0.004562376779827329,
    0.19589772566343958,
    0.0018307652855539633,
    0.0024373809976546406,
    0.0010910763243614134,
    0.0013851445384904116,
    0.011562478811383397,
    0.0014487344786067763,
    0.0008662875177938502,
    0.001081655530193219,
    0.0036211260670016426,
    0.0012005182597137926,
    0.0008206262574979196,
    0.0013884024958851027,
    0.00040275113364651385,
    0.0006003843042419653,
    0.0016611958951607308,
    0.0032439829291755987,
    0.002150418602406555,
    0.003655253274956507,
    0.00661945429342202,
    0.7233242444573894,
    0.006042151746046325,
    0.006278120533185768,
    0.0043802366148561886,
    0.004822847157549282
   ],
   "argmax": 23,
   "oracle_argmax": 23,
   "argmax_agrees": true,
   "selected_option": "fans",
   "oracle_selected_option": "fans",
   "oracle_margin": 0.5274265187939499,
   "abs_delta_p": [
    2.7002133971235076e-05,
    6.716686642898283e-06,
    3.559339247355842e-05,
    0.004924954918714364,
    2.2078762502466143e-05,
    1.911523295621085e-06,
    3.803752762962997e-06,
    1.4684895439159504e-06,
    0.0003108823453821536,
    4.585971392281158e-06,
    1.1924499910882726e-05,
    4.04753975654054e-06,
    0.00011919275906877459,
    4.326711382468583e-06,
    9.013770101921458e-08,
    2.313428070202907e-05,
    2.7659076058389053e-06,
    2.609963680315873e-06,
    7.836010477837562e-06,
    1.653683084648118e-05,
    3.584982496886548e-05,
    5.593050534835565e-05,
    0.00012064638669512847,
    0.004960603926504992,
    2.216773939077172e-05,
    6.695798024967427e-06,
    6.0749731244638325e-05,
    2.8265278136351164e-05
   ],
   "max_abs_delta_p": 0.004960603926504992,
   "mean_abs_delta_p": 0.0003865132788617041,
   "finite": true
  },
  {
   "id": "request-20-q01",
   "request_id": "request-20",
   "type": "choice",
   "zoo_only": true,
   "row_ids": [
    "request-20-q01-option-00",
    "request-20-q01-option-01",
    "request-20-q01-option-02",
    "request-20-q01-option-03",
    "request-20-q01-option-04",
    "request-20-q01-option-05",
    "request-20-q01-option-06",
    "request-20-q01-option-07",
    "request-20-q01-option-08",
    "request-20-q01-option-09",
    "request-20-q01-option-10",
    "request-20-q01-option-11",
    "request-20-q01-option-12",
    "request-20-q01-option-13",
    "request-20-q01-option-14",
    "request-20-q01-option-15",
    "request-20-q01-option-16",
    "request-20-q01-option-17",
    "request-20-q01-option-18",
    "request-20-q01-option-19",
    "request-20-q01-option-20",
    "request-20-q01-option-21",
    "request-20-q01-option-22",
    "request-20-q01-option-23",
    "request-20-q01-option-24",
    "request-20-q01-option-25",
    "request-20-q01-option-26",
    "request-20-q01-option-27",
    "request-20-q01-option-28",
    "request-20-q01-option-29",
    "request-20-q01-option-30",
    "request-20-q01-option-31"
   ],
   "options": [
    "lighting",
    "plumbing",
    "heating",
    "cooling",
    "doors",
    "windows",
    "floors",
    "walls",
    "ceilings",
    "roofs",
    "stairs",
    "lifts",
    "locks",
    "shelves",
    "desks",
    "chairs",
    "curtains",
    "carpets",
    "drains",
    "pipes",
    "valves",
    "tanks",
    "pumps",
    "fans",
    "sockets",
    "switches",
    "cables",
    "sensors",
    "meters",
    "signs",
    "paths",
    "gates"
   ],
   "nopts": 32,
   "scalars": [
    -7.8203125,
    -7.390625,
    -8.3125,
    -8.3125,
    2.388671875,
    -5.38671875,
    -8.9375,
    -6.91015625,
    -8.8828125,
    -8.15625,
    -9.2265625,
    -6.13671875,
    -1.23046875,
    -9.828125,
    -10.515625,
    -10.28125,
    -6.26953125,
    -10.765625,
    -8.09375,
    -7.0546875,
    -2.525390625,
    -7.37109375,
    -6.89453125,
    -8.109375,
    -8.0078125,
    -4.9921875,
    -5.60546875,
    -3.26953125,
    -8.1953125,
    -9.1875,
    -7.9296875,
    4.171875
   ],
   "scalars_oracle": [
    -7.80185079574585,
    -7.406404495239258,
    -8.328375816345215,
    -8.3722505569458,
    2.405498504638672,
    -5.341571807861328,
    -8.924745559692383,
    -6.8783721923828125,
    -8.876729011535645,
    -8.154253005981445,
    -9.206731796264648,
    -6.098209381103516,
    -1.189605712890625,
    -9.841769218444824,
    -10.507135391235352,
    -10.270621299743652,
    -6.233147144317627,
    -10.75239372253418,
    -8.108002662658691,
    -7.108526229858398,
    -2.5550146102905273,
    -7.36898946762085,
    -6.927596569061279,
    -8.134787559509277,
    -8.004584312438965,
    -4.9726715087890625,
    -5.6035261154174805,
    -3.2340099811553955,
    -8.21119499206543,
    -9.236793518066406,
    -7.896141052246094,
    4.17133903503418
   ],
   "p": [
    0.0007137425889557862,
    0.000912381388637161,
    0.0005387611717699894,
    0.0005387611717699894,
    0.24383029129633016,
    0.002867361011810947,
    0.0003769563960915068,
    0.0012006419647811455,
    0.0003889222765820326,
    0.0005890776991138169,
    0.00031956188524393545,
    0.0018679109551354167,
    0.030827036047220712,
    0.00022660330849782505,
    0.00015298560105902345,
    0.00017491014373200272,
    0.0017313956423677837,
    0.00013261983653806522,
    0.0006104963883636342,
    0.00110546614720029,
    0.014708664494548484,
    0.0009226212524415848,
    0.0012114099822957112,
    0.0006050697896905572,
    0.0006412244300499154,
    0.0035924647515875965,
    0.002530437211514992,
    0.009613934219165222,
    0.0005760743103397873,
    0.0003267751689412031,
    0.000670499111551375,
    0.6754949423566723
   ],
   "p_oracle": [
    0.0007191815249930839,
    0.0009015207946759931,
    0.0005323186915273457,
    0.0005191386905016756,
    0.2454589060345332,
    0.0029336055303064035,
    0.0003785922027948519,
    0.0012190362342608175,
    0.00038912384915785244,
    0.0005880083232525815,
    0.0003222490363386637,
    0.0019038304734785964,
    0.03146211983276343,
    0.00022417928797102953,
    0.00015327548882480683,
    0.00017545591105207873,
    0.0017625481986839066,
    0.0001332316396700548,
    0.0006037558356560152,
    0.0010688079711317084,
    0.014419055372251894,
    0.0009210028446555952,
    0.001185224679289517,
    0.0005945853163533855,
    0.0006405108561773736,
    0.003622021856556868,
    0.002525765151669818,
    0.009782091018493599,
    0.0005691834701096167,
    0.0003167606763826054,
    0.0006814573372852584,
    0.6732934558692004
   ],
   "argmax": 31,
   "oracle_argmax": 31,
   "argmax_agrees": true,
   "selected_option": "gates",
   "oracle_selected_option": "gates",
   "oracle_margin": 0.42783454983466723,
   "abs_delta_p": [
    5.438936037297664e-06,
    1.0860593961167808e-05,
    6.44248024264367e-06,
    1.9622481268313785e-05,
    0.001628614738203027,
    6.624451849545666e-05,
    1.6358067033451227e-06,
    1.8394269479672003e-05,
    2.0157257581983985e-07,
    1.0693758612353912e-06,
    2.687151094728278e-06,
    3.591951834317972e-05,
    0.0006350837855427192,
    2.4240205267955165e-06,
    2.8988776578338176e-07,
    5.45767320076011e-07,
    3.1152556316122945e-05,
    6.118031319895948e-07,
    6.740552707619018e-06,
    3.665817606858155e-05,
    0.0002896091222965892,
    1.618407785989543e-06,
    2.618530300619432e-05,
    1.0484473337171707e-05,
    7.13573872541777e-07,
    2.955710496927149e-05,
    4.672059845174225e-06,
    0.00016815679932837696,
    6.890840230170552e-06,
    1.0014492558597726e-05,
    1.0958225733883448e-05,
    0.002201486487471871
   ],
   "max_abs_delta_p": 0.002201486487471871,
   "mean_abs_delta_p": 0.00016471827756504392,
   "finite": true
  }
 ],
 "sessions": [
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-000-rows-a1",
   "mode": "int8lin",
   "indices": [
    0,
    1,
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9,
    10,
    11,
    12,
    13
   ],
   "pid": 2275,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:18:57+00:00",
   "phase": "complete",
   "runtime_calls": 5045,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 5045,
   "result": "PASS",
   "load_wall_seconds_contended": 7.048143124906346,
   "active_row": "request-01-q01-option-00",
   "active_step": 383,
   "reset_check": {
    "row": "request-01-q01-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": 4.328125,
    "final_scalar": 4.328125,
    "scalar_dtype": "<f2",
    "first_bits": "5444",
    "final_bits": "5444",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:21:30+00:00",
   "wall_seconds_contended": 152.8447280409746,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-000-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-000-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-000-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 153.6097169580171
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-001-rows-a1",
   "mode": "int8lin",
   "indices": [
    14,
    15,
    16,
    17,
    18,
    19,
    20,
    21,
    22,
    23,
    24,
    25,
    26,
    27
   ],
   "pid": 8988,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:21:31+00:00",
   "phase": "complete",
   "runtime_calls": 456,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 456,
   "result": "PASS",
   "load_wall_seconds_contended": 15.093804250005633,
   "active_row": "request-07-q01-option-02",
   "active_step": 25,
   "reset_check": {
    "row": "request-07-q01-option-02",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -9.3125,
    "final_scalar": -9.3125,
    "scalar_dtype": "<f2",
    "first_bits": "a8c8",
    "final_bits": "a8c8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:22:12+00:00",
   "wall_seconds_contended": 41.8093340001069,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-001-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-001-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-001-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 42.79751641699113
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-002-rows-a1",
   "mode": "int8lin",
   "indices": [
    28,
    29,
    30,
    31,
    32,
    33,
    34,
    35,
    36,
    37,
    38,
    39,
    40,
    41
   ],
   "pid": 10903,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:22:14+00:00",
   "phase": "complete",
   "runtime_calls": 465,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 465,
   "result": "PASS",
   "load_wall_seconds_contended": 0.9102606660453603,
   "active_row": "request-07-q03-option-05",
   "active_step": 37,
   "reset_check": {
    "row": "request-07-q03-option-05",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": 0.72998046875,
    "final_scalar": 0.72998046875,
    "scalar_dtype": "<f2",
    "first_bits": "d739",
    "final_bits": "d739",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:22:34+00:00",
   "wall_seconds_contended": 20.519833666970953,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-002-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-002-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-002-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 21.52754824992735
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-003-rows-a1",
   "mode": "int8lin",
   "indices": [
    42,
    43,
    44,
    45,
    46,
    47,
    48,
    49,
    50,
    51,
    52,
    53,
    54,
    55
   ],
   "pid": 11834,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:22:35+00:00",
   "phase": "complete",
   "runtime_calls": 553,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 553,
   "result": "PASS",
   "load_wall_seconds_contended": 1.8534544580616057,
   "active_row": "request-08-q01-option-09",
   "active_step": 26,
   "reset_check": {
    "row": "request-08-q01-option-09",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -10.6796875,
    "final_scalar": -10.6796875,
    "scalar_dtype": "<f2",
    "first_bits": "57c9",
    "final_bits": "57c9",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:22:55+00:00",
   "wall_seconds_contended": 19.880605124984868,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-003-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-003-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-003-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 20.728382542030886
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-004-rows-a1",
   "mode": "int8lin",
   "indices": [
    56,
    57,
    58,
    59,
    60,
    61,
    62,
    63,
    64,
    65,
    66,
    67,
    68,
    69
   ],
   "pid": 12712,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:22:56+00:00",
   "phase": "complete",
   "runtime_calls": 398,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 398,
   "result": "PASS",
   "load_wall_seconds_contended": 5.96308279200457,
   "active_row": "request-09-q01-option-01",
   "active_step": 25,
   "reset_check": {
    "row": "request-09-q01-option-01",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -7.5546875,
    "final_scalar": -7.5546875,
    "scalar_dtype": "<f2",
    "first_bits": "8ec7",
    "final_bits": "8ec7",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:23:25+00:00",
   "wall_seconds_contended": 29.613253624993376,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-004-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-004-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-004-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 30.53067129093688
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-005-rows-a1",
   "mode": "int8lin",
   "indices": [
    70,
    71,
    72,
    73,
    74,
    75,
    76,
    77,
    78,
    79,
    80,
    81,
    82,
    83
   ],
   "pid": 14020,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:23:26+00:00",
   "phase": "complete",
   "runtime_calls": 393,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 393,
   "result": "PASS",
   "load_wall_seconds_contended": 1.038600540952757,
   "active_row": "request-10-q01-option-00",
   "active_step": 25,
   "reset_check": {
    "row": "request-10-q01-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": 0.86083984375,
    "final_scalar": 0.86083984375,
    "scalar_dtype": "<f2",
    "first_bits": "e33a",
    "final_bits": "e33a",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:23:49+00:00",
   "wall_seconds_contended": 22.825386416981928,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-005-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-005-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-005-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 23.692003582953475
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-006-rows-a1",
   "mode": "int8lin",
   "indices": [
    84,
    85,
    86,
    87,
    88,
    89,
    90,
    91,
    92,
    93,
    94,
    95,
    96,
    97
   ],
   "pid": 15007,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:23:50+00:00",
   "phase": "complete",
   "runtime_calls": 410,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 410,
   "result": "PASS",
   "load_wall_seconds_contended": 1.0397417919011787,
   "active_row": "request-10-q03-option-00",
   "active_step": 29,
   "reset_check": {
    "row": "request-10-q03-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -2.60546875,
    "final_scalar": -2.60546875,
    "scalar_dtype": "<f2",
    "first_bits": "36c1",
    "final_bits": "36c1",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:24:13+00:00",
   "wall_seconds_contended": 23.54295637505129,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-006-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-006-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-006-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 24.396994665963575
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-007-rows-a1",
   "mode": "int8lin",
   "indices": [
    98,
    99,
    100,
    101,
    102,
    103,
    104,
    105,
    106,
    107,
    108,
    109,
    110,
    111
   ],
   "pid": 16070,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:24:14+00:00",
   "phase": "complete",
   "runtime_calls": 407,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 407,
   "result": "PASS",
   "load_wall_seconds_contended": 1.034129999927245,
   "active_row": "request-11-q01-option-12",
   "active_step": 26,
   "reset_check": {
    "row": "request-11-q01-option-12",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -9.3359375,
    "final_scalar": -9.3359375,
    "scalar_dtype": "<f2",
    "first_bits": "abc8",
    "final_bits": "abc8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:24:38+00:00",
   "wall_seconds_contended": 23.4051565839909,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-007-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-007-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-007-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 24.232961374917068
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-008-rows-a1",
   "mode": "int8lin",
   "indices": [
    112,
    113,
    114,
    115,
    116,
    117,
    118,
    119,
    120,
    121,
    122,
    123,
    124,
    125
   ],
   "pid": 17068,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:24:39+00:00",
   "phase": "complete",
   "runtime_calls": 421,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 421,
   "result": "PASS",
   "load_wall_seconds_contended": 1.0418234589742497,
   "active_row": "request-12-q01-option-07",
   "active_step": 27,
   "reset_check": {
    "row": "request-12-q01-option-07",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -2.654296875,
    "final_scalar": -2.654296875,
    "scalar_dtype": "<f2",
    "first_bits": "4fc1",
    "final_bits": "4fc1",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:25:03+00:00",
   "wall_seconds_contended": 23.97896324994508,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-008-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-008-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-008-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 24.842785416985862
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-009-rows-a1",
   "mode": "int8lin",
   "indices": [
    126,
    127,
    128,
    129,
    130,
    131,
    132,
    133,
    134,
    135,
    136,
    137,
    138,
    139
   ],
   "pid": 18164,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:25:03+00:00",
   "phase": "complete",
   "runtime_calls": 460,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 460,
   "result": "PASS",
   "load_wall_seconds_contended": 0.8195149169769138,
   "active_row": "request-13-q01-option-00",
   "active_step": 30,
   "reset_check": {
    "row": "request-13-q01-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": 4.39453125,
    "final_scalar": 4.39453125,
    "scalar_dtype": "<f2",
    "first_bits": "6544",
    "final_bits": "6544",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:25:19+00:00",
   "wall_seconds_contended": 15.344809166970663,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-009-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-009-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-009-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 15.973523750086315
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-010-rows-a1",
   "mode": "int8lin",
   "indices": [
    140,
    141,
    142,
    143,
    144,
    145,
    146,
    147,
    148,
    149,
    150,
    151,
    152,
    153
   ],
   "pid": 18815,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:25:19+00:00",
   "phase": "complete",
   "runtime_calls": 455,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 455,
   "result": "PASS",
   "load_wall_seconds_contended": 0.8419320000102744,
   "active_row": "request-14-q02-option-00",
   "active_step": 31,
   "reset_check": {
    "row": "request-14-q02-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -0.489013671875,
    "final_scalar": -0.489013671875,
    "scalar_dtype": "<f2",
    "first_bits": "d3b7",
    "final_bits": "d3b7",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:25:35+00:00",
   "wall_seconds_contended": 15.293512583943084,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-010-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-010-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-010-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 15.897724165930413
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-011-rows-a1",
   "mode": "int8lin",
   "indices": [
    154,
    155,
    156,
    157,
    158,
    159,
    160,
    161,
    162,
    163,
    164,
    165,
    166,
    167
   ],
   "pid": 19465,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:25:35+00:00",
   "phase": "complete",
   "runtime_calls": 452,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 452,
   "result": "PASS",
   "load_wall_seconds_contended": 0.8343904579523951,
   "active_row": "request-15-q03-option-00",
   "active_step": 29,
   "reset_check": {
    "row": "request-15-q03-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -1.990234375,
    "final_scalar": -1.990234375,
    "scalar_dtype": "<f2",
    "first_bits": "f6bf",
    "final_bits": "f6bf",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:25:51+00:00",
   "wall_seconds_contended": 15.28087899996899,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-011-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-011-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-011-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 15.916843333048746
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-012-rows-a1",
   "mode": "int8lin",
   "indices": [
    168,
    169,
    170,
    171,
    172,
    173,
    174,
    175,
    176,
    177,
    178,
    179,
    180,
    181
   ],
   "pid": 20146,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:25:51+00:00",
   "phase": "complete",
   "runtime_calls": 437,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 437,
   "result": "PASS",
   "load_wall_seconds_contended": 0.8515001250198111,
   "active_row": "request-16-q03-option-01",
   "active_step": 33,
   "reset_check": {
    "row": "request-16-q03-option-01",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -13.2109375,
    "final_scalar": -13.2109375,
    "scalar_dtype": "<f2",
    "first_bits": "9bca",
    "final_bits": "9bca",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:26:06+00:00",
   "wall_seconds_contended": 15.170597542077303,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-012-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-012-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-012-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 15.82111629191786
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-013-rows-a1",
   "mode": "int8lin",
   "indices": [
    182,
    183,
    184,
    185,
    186,
    187,
    188,
    189,
    190,
    191,
    192,
    193,
    194,
    195
   ],
   "pid": 20874,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:26:07+00:00",
   "phase": "complete",
   "runtime_calls": 387,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 387,
   "result": "PASS",
   "load_wall_seconds_contended": 0.93780433293432,
   "active_row": "request-17-q01-option-06",
   "active_step": 25,
   "reset_check": {
    "row": "request-17-q01-option-06",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -8.515625,
    "final_scalar": -8.515625,
    "scalar_dtype": "<f2",
    "first_bits": "42c8",
    "final_bits": "42c8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:26:23+00:00",
   "wall_seconds_contended": 15.88551587506663,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-013-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-013-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-013-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 16.60856729093939
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-014-rows-a1",
   "mode": "int8lin",
   "indices": [
    196,
    197,
    198,
    199,
    200,
    201,
    202,
    203,
    204,
    205,
    206,
    207,
    208,
    209
   ],
   "pid": 21553,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:26:24+00:00",
   "phase": "complete",
   "runtime_calls": 415,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 415,
   "result": "PASS",
   "load_wall_seconds_contended": 2.1805567919509485,
   "active_row": "request-18-q01-option-00",
   "active_step": 27,
   "reset_check": {
    "row": "request-18-q01-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -8.0703125,
    "final_scalar": -8.0703125,
    "scalar_dtype": "<f2",
    "first_bits": "09c8",
    "final_bits": "09c8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:27:05+00:00",
   "wall_seconds_contended": 41.0307866249932,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-014-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-014-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-014-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 41.778699416085146
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-015-rows-a1",
   "mode": "int8lin",
   "indices": [
    210,
    211,
    212,
    213,
    214,
    215,
    216,
    217,
    218,
    219,
    220,
    221,
    222,
    223
   ],
   "pid": 23384,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:27:05+00:00",
   "phase": "complete",
   "runtime_calls": 411,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 411,
   "result": "PASS",
   "load_wall_seconds_contended": 14.978590750019066,
   "active_row": "request-18-q01-option-14",
   "active_step": 27,
   "reset_check": {
    "row": "request-18-q01-option-14",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -9.6953125,
    "final_scalar": -9.6953125,
    "scalar_dtype": "<f2",
    "first_bits": "d9c8",
    "final_bits": "d9c8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:27:35+00:00",
   "wall_seconds_contended": 29.885863124975003,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-015-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-015-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-015-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 30.622475333046168
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-016-rows-a1",
   "mode": "int8lin",
   "indices": [
    224,
    225,
    226,
    227,
    228,
    229,
    230,
    231,
    232,
    233,
    234,
    235,
    236,
    237
   ],
   "pid": 24674,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:27:36+00:00",
   "phase": "complete",
   "runtime_calls": 385,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 385,
   "result": "PASS",
   "load_wall_seconds_contended": 1.105354291968979,
   "active_row": "request-19-q01-option-04",
   "active_step": 24,
   "reset_check": {
    "row": "request-19-q01-option-04",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -9.203125,
    "final_scalar": -9.203125,
    "scalar_dtype": "<f2",
    "first_bits": "9ac8",
    "final_bits": "9ac8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:27:52+00:00",
   "wall_seconds_contended": 16.385647875024006,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-016-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-016-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-016-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 17.09001583303325
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-017-rows-a1",
   "mode": "int8lin",
   "indices": [
    238,
    239,
    240,
    241,
    242,
    243,
    244,
    245,
    246,
    247,
    248,
    249,
    250,
    251
   ],
   "pid": 25400,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:27:53+00:00",
   "phase": "complete",
   "runtime_calls": 383,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 383,
   "result": "PASS",
   "load_wall_seconds_contended": 1.492397958994843,
   "active_row": "request-19-q01-option-18",
   "active_step": 25,
   "reset_check": {
    "row": "request-19-q01-option-18",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -9.34375,
    "final_scalar": -9.34375,
    "scalar_dtype": "<f2",
    "first_bits": "acc8",
    "final_bits": "acc8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:28:17+00:00",
   "wall_seconds_contended": 23.861984040937386,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-017-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-017-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-017-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 24.774880500044674
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-018-rows-a1",
   "mode": "int8lin",
   "indices": [
    252,
    253,
    254,
    255,
    256,
    257,
    258,
    259,
    260,
    261,
    262,
    263,
    264,
    265
   ],
   "pid": 26462,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:28:18+00:00",
   "phase": "complete",
   "runtime_calls": 370,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 370,
   "result": "PASS",
   "load_wall_seconds_contended": 1.063269458943978,
   "active_row": "request-20-q01-option-04",
   "active_step": 23,
   "reset_check": {
    "row": "request-20-q01-option-04",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": 2.388671875,
    "final_scalar": 2.388671875,
    "scalar_dtype": "<f2",
    "first_bits": "c740",
    "final_bits": "c740",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:28:40+00:00",
   "wall_seconds_contended": 21.934233875013888,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-018-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-018-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-018-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 22.802352083963342
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-019-rows-a1",
   "mode": "int8lin",
   "indices": [
    266,
    267,
    268,
    269,
    270,
    271,
    272,
    273,
    274,
    275,
    276,
    277,
    278,
    279
   ],
   "pid": 27411,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:28:41+00:00",
   "phase": "complete",
   "runtime_calls": 371,
   "row_executions": 15,
   "planned_row_executions": 15,
   "planned_runtime_calls": 371,
   "result": "PASS",
   "load_wall_seconds_contended": 1.0570054579293355,
   "active_row": "request-20-q01-option-18",
   "active_step": 24,
   "reset_check": {
    "row": "request-20-q01-option-18",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": -8.09375,
    "final_scalar": -8.09375,
    "scalar_dtype": "<f2",
    "first_bits": "0cc8",
    "final_bits": "0cc8",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:29:03+00:00",
   "wall_seconds_contended": 22.012059709057212,
   "kind": "rows",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-019-rows-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-019-rows-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-019-rows-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 22.891407417017035
  },
  {
   "schema": "coreai-scalar-readout-session/1",
   "session": "int8lin-020-global_reset-a1",
   "mode": "int8lin",
   "indices": [
    0
   ],
   "pid": 28408,
   "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
   "snapshot": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/merged",
   "license": "cc-by-nc-4.0",
   "started_at": "2026-09-23T01:29:04+00:00",
   "phase": "complete",
   "runtime_calls": 768,
   "row_executions": 2,
   "planned_row_executions": 2,
   "planned_runtime_calls": 768,
   "result": "PASS",
   "load_wall_seconds_contended": 1.056914584012702,
   "active_row": "request-01-q01-option-00",
   "active_step": 383,
   "reset_check": {
    "row": "request-01-q01-option-00",
    "identical": true,
    "comparison": "dtype and exact bytes",
    "abs_diff": 0.0,
    "first_scalar": 4.328125,
    "final_scalar": 4.328125,
    "scalar_dtype": "<f2",
    "first_bits": "5444",
    "final_bits": "5444",
    "fresh_state_each_row": true,
    "same_process": true
   },
   "completed_at": "2026-09-23T01:29:42+00:00",
   "wall_seconds_contended": 38.47855954198167,
   "kind": "global_reset",
   "transcript": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-020-global_reset-a1.json",
   "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/readout-int8lin-020-global_reset-a1.log",
   "command": [
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/.venv/bin/python",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/zoo/conversion/scalar/readout_gate_scalar.py",
    "--worker",
    "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/results/readout_sessions/int8lin-020-global_reset-a1.config.json"
   ],
   "returncode": 0,
   "timed_out": false,
   "process_wall_seconds_contended": 39.365898541989736
  }
 ],
 "result": "PASS",
 "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
 "aot": {
  "source": {
   "path": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/exports/system_one_qwen3_5_4b_scorer_decode_int8lin/system_one_qwen3_5_4b_scorer_decode_int8lin.aimodel",
   "bytes": 160,
   "mtime_ns": 1790114434647811488
  },
  "asset": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc/system_one_qwen3_5_4b_scorer_decode_int8lin.h16c.aimodelc",
  "command": [
   "/var/run/com.apple.security.cryptexd/mnt/com.apple.MobileAsset.MetalToolchain-v27.1.266.1.9a6Ccy/Metal.xctoolchain/usr/bin/coreai-build",
   "compile",
   "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/exports/system_one_qwen3_5_4b_scorer_decode_int8lin/system_one_qwen3_5_4b_scorer_decode_int8lin.aimodel",
   "--output",
   "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/aotc",
   "--platform",
   "macOS",
   "--preferred-compute",
   "gpu",
   "--architecture",
   "h16c",
   "--expect-frequent-reshapes"
  ],
  "flags": [
   "--platform",
   "macOS",
   "--preferred-compute",
   "gpu",
   "--architecture",
   "h16c",
   "--expect-frequent-reshapes"
  ],
  "license": "cc-by-nc-4.0",
  "log": "/Users/majimadaisuke/code/codex-conversions/2026-09-23/pngwn-scorer-4b-coreai/results/round3/scalar-work/runtime/logs/aot-int8lin.log",
  "started_at": "2026-09-23T00:54:10+00:00",
  "result": "PASS",
  "reused": true,
  "returncode": 0,
  "wall_seconds_contended": 192.11521258403081,
  "completed_at": "2026-09-23T00:57:22+00:00"
 },
 "summary": {
  "rows": 280,
  "expected_rows": 280,
  "questions": 48,
  "expected_questions": 48,
  "argmax_agreement": 48,
  "margin_ge_0_02_questions": 46,
  "margin_ge_0_02_argmax_agreement": 46,
  "near_ties": [
   {
    "id": "request-07-q03",
    "oracle_margin": 0.006739843342914137,
    "oracle_argmax": 1,
    "argmax": 1,
    "argmax_agrees": true,
    "p": [
     0.0668905628612553,
     0.13881996412774117,
     0.13353910941541403,
     0.11781497336516934,
     0.0988923283790044,
     0.11531078903673647,
     0.11923700360205998,
     0.11317517052682867,
     0.06733529674798773,
     0.028984801937802986
    ],
    "p_oracle": [
     0.06844557046678758,
     0.1410972557943501,
     0.13435741245143595,
     0.11769923201661937,
     0.09850107499890995,
     0.11399026048335885,
     0.11797473484160344,
     0.11195394081789756,
     0.0672594693782866,
     0.028721048750750485
    ]
   },
   {
    "id": "request-08-q03",
    "oracle_margin": 0.004176430865158717,
    "oracle_argmax": 5,
    "argmax": 5,
    "argmax_agrees": true,
    "p": [
     0.03844735559601341,
     0.08568083778528952,
     0.11268820304996982,
     0.11529616943900456,
     0.1093431181851565,
     0.12812072124379084,
     0.12328126965086839,
     0.11652512124167493,
     0.1072282308349132,
     0.06338897297331891
    ],
    "p_oracle": [
     0.03829370334805408,
     0.08559956498729611,
     0.11225994259558048,
     0.11473061408817004,
     0.10857444261658075,
     0.12713889005910925,
     0.12296245919395053,
     0.11719791061365697,
     0.10862090712691694,
     0.06462156537068481
    ]
   }
  ],
  "strict_disagreements": [],
  "max_abs_delta_p": 0.010874463417753888,
  "mean_of_question_mean_abs_delta_p": 0.0008333858900326634,
  "mean_of_row_mean_abs_delta_p": 0.0008333858900326634,
  "mean_of_row_mean_note": "Each probability row is one question; options are weighted equally within it.",
  "mean_option_abs_delta_p": 0.0004889207254776194,
  "max_abs_delta_scalar": 0.0973968505859375,
  "mean_abs_delta_scalar": 0.021620490508420128,
  "scalar_min": -14.59375,
  "scalar_max": 7.6484375,
  "scalar_absmax": 14.59375,
  "oracle_scalar_min": -14.593600273132324,
  "oracle_scalar_max": 7.658699989318848,
  "oracle_scalar_absmax": 14.593600273132324,
  "finite_all": true,
  "reset_identical": true,
  "session_resets_identical": true,
  "max_row_executions_per_process": 15,
  "provisional_max_delta_p_expectation": 0.05,
  "provisional_expectation_exceeded": false,
  "provisional_expectation_is_acceptance_gate": false,
  "worst_five_questions": [
   {
    "id": "request-14-q04",
    "request_id": "request-14",
    "type": "noul",
    "zoo_only": false,
    "row_ids": [
     "request-14-q04-option-00",
     "request-14-q04-option-01"
    ],
    "options": [
     "yes",
     "no"
    ],
    "nopts": 2,
    "scalars": [
     -1.6494140625,
     1.400390625
    ],
    "scalars_oracle": [
     -1.5531220436096191,
     1.3508743047714233
    ],
    "p": [
     0.14896453614240898,
     0.851035463857591
    ],
    "p_oracle": [
     0.15983899956016281,
     0.8401610004398371
    ],
    "argmax": 1,
    "oracle_argmax": 1,
    "argmax_agrees": true,
    "selected_option": "no",
    "oracle_selected_option": "no",
    "oracle_margin": 0.6803220008796743,
    "abs_delta_p": [
     0.010874463417753832,
     0.010874463417753888
    ],
    "max_abs_delta_p": 0.010874463417753888,
    "mean_abs_delta_p": 0.01087446341775386,
    "finite": true
   },
   {
    "id": "request-11-q03",
    "request_id": "request-11",
    "type": "score",
    "zoo_only": false,
    "row_ids": [
     "request-11-q03-option-00",
     "request-11-q03-option-01",
     "request-11-q03-option-02"
    ],
    "options": [
     "none",
     "some",
     "complete"
    ],
    "nopts": 3,
    "scalars": [
     -1.96875,
     -0.794921875,
     0.211669921875
    ],
    "scalars_oracle": [
     -1.9961133003234863,
     -0.7945132851600647,
     0.13405287265777588
    ],
    "p": [
     0.15547332793355584,
     0.30406245395135767,
     0.5404642181150865
    ],
    "p_oracle": [
     0.15711289848380908,
     0.3121841461234784,
     0.5307029553927125
    ],
    "argmax": 2,
    "oracle_argmax": 2,
    "argmax_agrees": true,
    "selected_option": "complete",
    "oracle_selected_option": "complete",
    "oracle_margin": 0.21851880926923412,
    "abs_delta_p": [
     0.0016395705502532387,
     0.00812169217212072,
     0.009761262722373987
    ],
    "max_abs_delta_p": 0.009761262722373987,
    "mean_abs_delta_p": 0.006507508481582648,
    "finite": true
   },
   {
    "id": "request-15-q04",
    "request_id": "request-15",
    "type": "noul",
    "zoo_only": false,
    "row_ids": [
     "request-15-q04-option-00",
     "request-15-q04-option-01"
    ],
    "options": [
     "yes",
     "no"
    ],
    "nopts": 2,
    "scalars": [
     -1.669921875,
     1.3291015625
    ],
    "scalars_oracle": [
     -1.7337946891784668,
     1.3656667470932007
    ],
    "p": [
     0.1526808435988674,
     0.8473191564011325
    ],
    "p_oracle": [
     0.14540299930898767,
     0.8545970006910122
    ],
    "argmax": 1,
    "oracle_argmax": 1,
    "argmax_agrees": true,
    "selected_option": "no",
    "oracle_selected_option": "no",
    "oracle_margin": 0.7091940013820245,
    "abs_delta_p": [
     0.007277844289879737,
     0.007277844289879654
    ],
    "max_abs_delta_p": 0.007277844289879737,
    "mean_abs_delta_p": 0.007277844289879695,
    "finite": true
   },
   {
    "id": "request-13-q05",
    "request_id": "request-13",
    "type": "score",
    "zoo_only": false,
    "row_ids": [
     "request-13-q05-option-00",
     "request-13-q05-option-01",
     "request-13-q05-option-02",
     "request-13-q05-option-03"
    ],
    "options": [
     "unusable",
     "limited",
     "mostly usable",
     "fully usable"
    ],
    "nopts": 4,
    "scalars": [
     1.6171875,
     0.30712890625,
     -1.2392578125,
     -3.888671875
    ],
    "scalars_oracle": [
     1.6196714639663696,
     0.27900874614715576,
     -1.3120677471160889,
     -3.97291898727417
    ],
    "p": [
     0.5842727542749888,
     0.2763762074563208,
     0.11421851986295291,
     0.02513251840573747
    ],
    "p_oracle": [
     0.5906616660481768,
     0.274554667682575,
     0.11060483754235892,
     0.024178828726889312
    ],
    "argmax": 0,
    "oracle_argmax": 0,
    "argmax_agrees": true,
    "selected_option": "unusable",
    "oracle_selected_option": "unusable",
    "oracle_margin": 0.3161069983656018,
    "abs_delta_p": [
     0.006388911773188011,
     0.0018215397737457728,
     0.003613682320593989,
     0.0009536896788481589
    ],
    "max_abs_delta_p": 0.006388911773188011,
    "mean_abs_delta_p": 0.003194455886593983,
    "finite": true
   },
   {
    "id": "request-19-q01",
    "request_id": "request-19",
    "type": "choice",
    "zoo_only": true,
    "row_ids": [
     "request-19-q01-option-00",
     "request-19-q01-option-01",
     "request-19-q01-option-02",
     "request-19-q01-option-03",
     "request-19-q01-option-04",
     "request-19-q01-option-05",
     "request-19-q01-option-06",
     "request-19-q01-option-07",
     "request-19-q01-option-08",
     "request-19-q01-option-09",
     "request-19-q01-option-10",
     "request-19-q01-option-11",
     "request-19-q01-option-12",
     "request-19-q01-option-13",
     "request-19-q01-option-14",
     "request-19-q01-option-15",
     "request-19-q01-option-16",
     "request-19-q01-option-17",
     "request-19-q01-option-18",
     "request-19-q01-option-19",
     "request-19-q01-option-20",
     "request-19-q01-option-21",
     "request-19-q01-option-22",
     "request-19-q01-option-23",
     "request-19-q01-option-24",
     "request-19-q01-option-25",
     "request-19-q01-option-26",
     "request-19-q01-option-27"
    ],
    "options": [
     "lighting",
     "plumbing",
     "heating",
     "cooling",
     "doors",
     "windows",
     "floors",
     "walls",
     "ceilings",
     "roofs",
     "stairs",
     "lifts",
     "locks",
     "shelves",
     "desks",
     "chairs",
     "curtains",
     "carpets",
     "drains",
     "pipes",
     "valves",
     "tanks",
     "pumps",
     "fans",
     "sockets",
     "switches",
     "cables",
     "sensors"
    ],
    "nopts": 28,
    "scalars": [
     -7.59765625,
     -8.25,
     -7.59765625,
     -0.9609375,
     -9.203125,
     -8.6796875,
     -10.09375,
     -9.671875,
     -5.91015625,
     -9.5859375,
     -10.515625,
     -10.109375,
     -8.046875,
     -9.9140625,
     -10.5859375,
     -9.6953125,
     -11.84375,
     -11.140625,
     -9.34375,
     -8.171875,
     -8.9296875,
     -7.9453125,
     -6.96484375,
     1.26953125,
     -7.0859375,
     -7.0234375,
     -7.6796875,
     -7.4765625
    ],
    "scalars_oracle": [
     -7.635459899902344,
     -8.273568153381348,
     -7.611283779144287,
     -1.0317230224609375,
     -9.20922565460205,
     -8.708393096923828,
     -10.114972114562988,
     -9.697352409362793,
     -5.983921051025391,
     -9.618802070617676,
     -10.518702507019043,
     -10.130147933959961,
     -8.015636444091797,
     -9.947691917419434,
     -10.613463401794434,
     -9.693241119384766,
     -11.859024047851562,
     -11.160334587097168,
     -9.379319190979004,
     -8.208106994628906,
     -8.927600860595703,
     -7.999220848083496,
     -6.959987640380859,
     1.2542405128479004,
     -7.119679927825928,
     -7.052636623382568,
     -7.682580471038818,
     -7.514122486114502
    ],
    "p": [
     0.004526783387353771,
     0.0031181621105346538,
     0.004526783387353771,
     0.20082268058215394,
     0.0018086865230514972,
     0.0024392925209502617,
     0.0010872725715984505,
     0.0013836760489464956,
     0.011873361156765551,
     0.0014533204499990575,
     0.0008543630178829675,
     0.0010776079904366784,
     0.003501933307932868,
     0.0012048449710962612,
     0.0008207163951989388,
     0.0013652682151830736,
     0.00039998522604067495,
     0.0005977743405616494,
     0.0016690319056385683,
     0.00326051976002208,
     0.0021145687774376896,
     0.0037111837803048627,
     0.006498807906726891,
     0.7183636405308844,
     0.006064319485437097,
     0.006284816331210735,
     0.00431948688361155,
     0.004851112435685633
    ],
    "p_oracle": [
     0.004499781253382536,
     0.003124878797177552,
     0.004562376779827329,
     0.19589772566343958,
     0.0018307652855539633,
     0.0024373809976546406,
     0.0010910763243614134,
     0.0013851445384904116,
     0.011562478811383397,
     0.0014487344786067763,
     0.0008662875177938502,
     0.001081655530193219,
     0.0036211260670016426,
     0.0012005182597137926,
     0.0008206262574979196,
     0.0013884024958851027,
     0.00040275113364651385,
     0.0006003843042419653,
     0.0016611958951607308,
     0.0032439829291755987,
     0.002150418602406555,
     0.003655253274956507,
     0.00661945429342202,
     0.7233242444573894,
     0.006042151746046325,
     0.006278120533185768,
     0.0043802366148561886,
     0.004822847157549282
    ],
    "argmax": 23,
    "oracle_argmax": 23,
    "argmax_agrees": true,
    "selected_option": "fans",
    "oracle_selected_option": "fans",
    "oracle_margin": 0.5274265187939499,
    "abs_delta_p": [
     2.7002133971235076e-05,
     6.716686642898283e-06,
     3.559339247355842e-05,
     0.004924954918714364,
     2.2078762502466143e-05,
     1.911523295621085e-06,
     3.803752762962997e-06,
     1.4684895439159504e-06,
     0.0003108823453821536,
     4.585971392281158e-06,
     1.1924499910882726e-05,
     4.04753975654054e-06,
     0.00011919275906877459,
     4.326711382468583e-06,
     9.013770101921458e-08,
     2.313428070202907e-05,
     2.7659076058389053e-06,
     2.609963680315873e-06,
     7.836010477837562e-06,
     1.653683084648118e-05,
     3.584982496886548e-05,
     5.593050534835565e-05,
     0.00012064638669512847,
     0.004960603926504992,
     2.216773939077172e-05,
     6.695798024967427e-06,
     6.0749731244638325e-05,
     2.8265278136351164e-05
    ],
    "max_abs_delta_p": 0.004960603926504992,
    "mean_abs_delta_p": 0.0003865132788617041,
    "finite": true
   }
  ]
 },
 "reset_check": {
  "row": "request-01-q01-option-00",
  "identical": true,
  "comparison": "dtype and exact bytes",
  "abs_diff": 0.0,
  "first_scalar": 4.328125,
  "final_scalar": 4.328125,
  "first_bits": "5444",
  "final_bits": "5444",
  "final_session": "int8lin-020-global_reset-a1",
  "cross_process": true,
  "local_resets_in_every_session": true
 },
 "wall_seconds_contended": 645.9986511670286,
 "completed_at": "2026-09-23T01:29:43+00:00"
}