{
  "experiment": "H1 real-text conditional byte-model pilot v8 (three-level taxonomy, 16 categories)",
  "completed_utc": "2026-11-18T01:37:14.080540+00:00",
  "smoke_test_not_protocol": false,
  "protocol": {
    "dataset": "SetFit/20_newsgroups",
    "revision": "f1b91292074e7cfb69be58b642d583ec262f30ed",
    "dataset_split": "train only; official test unused",
    "selected_data_sha256": "217f4a3c2d1161c816e439183b609074e6fcaa822b9066210c990a7e11c90974",
    "categories": [
      "comp.sys.ibm.pc.hardware",
      "comp.sys.mac.hardware",
      "comp.graphics",
      "comp.os.ms-windows.misc",
      "rec.autos",
      "rec.motorcycles",
      "rec.sport.baseball",
      "rec.sport.hockey",
      "sci.crypt",
      "sci.electronics",
      "sci.med",
      "sci.space",
      "talk.politics.guns",
      "talk.politics.mideast",
      "alt.atheism",
      "soc.religion.christian"
    ],
    "taxonomy": "top = c // 4, mid = (c // 2) % 2, leaf = c % 2; prefix is exactly 3 tokens",
    "train_per_category": 300,
    "validation_per_category": 80,
    "validation_messages": 1280,
    "bytes_per_example": 96,
    "seed_values": [
      2027,
      3037,
      4049,
      5051,
      6067
    ],
    "steps": [
      200,
      400,
      800
    ],
    "total_updates": 800,
    "batch_size": 24,
    "conditions": [
      "none",
      "flat",
      "top_only",
      "top_mid",
      "hierarchy",
      "scrambled_1",
      "scrambled_2",
      "scrambled_3",
      "scrambled_4",
      "scrambled_5"
    ],
    "permutation_seeds_selected": [
      31415,
      27182,
      16180,
      14142,
      17320
    ],
    "permutation_seeds_rejected": [],
    "permutation_seeds_primary": [
      31415,
      27182,
      16180,
      14142,
      17320
    ],
    "permutation_seeds_fallback": [
      11235,
      13579,
      24680,
      97531
    ],
    "scrambled_permutations": {
      "scrambled_1": [
        10,
        12,
        13,
        4,
        15,
        1,
        3,
        5,
        2,
        6,
        11,
        8,
        0,
        7,
        14,
        9
      ],
      "scrambled_2": [
        13,
        12,
        2,
        5,
        4,
        7,
        8,
        14,
        11,
        3,
        1,
        10,
        15,
        6,
        0,
        9
      ],
      "scrambled_3": [
        6,
        2,
        10,
        12,
        5,
        14,
        11,
        7,
        1,
        8,
        0,
        4,
        9,
        15,
        13,
        3
      ],
      "scrambled_4": [
        1,
        4,
        8,
        9,
        6,
        12,
        3,
        7,
        11,
        13,
        0,
        14,
        10,
        5,
        15,
        2
      ],
      "scrambled_5": [
        8,
        13,
        4,
        3,
        14,
        11,
        0,
        9,
        2,
        15,
        5,
        10,
        7,
        12,
        1,
        6
      ]
    },
    "permutation_rejection_rule": "rejected if >= 6 of the 24 same-family pairs (families {0-3},{4-7},{8-11},{12-15}) share the same code // 4 (identity and any top-partition-preserving permutation give 24)",
    "same_batch_sequence_per_seed_across_conditions": true,
    "evaluations": "each checkpoint: (a) eval_nll_labeled with the condition label prefix; (b) eval_nll_ablated with dummy prefix [256, 256, 256] for all conditions; label_use_gap = ablated - labeled",
    "metric": "validation NLL in nats per byte, lower is better",
    "model": "randomly initialized 2-layer transformer width 64, 4 heads, FF 128",
    "limitations": "five seeds, five scrambled permutations, one fixed 4 x 2 x 2 taxonomy (the mid and leaf splits are a judgement call), short prefixes, tiny model (800 updates), one historical text dataset (20 Newsgroups); no significance test is run or claimed; no data-savings percentage is estimated; labels present at labeled evaluation; a null result is informative; not a pretrained-LLM or general data-savings result"
  },
  "environment": {
    "device": "cuda",
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition MIG 2g.48gb",
    "torch": "2.13.0+cu130",
    "cuda": "13.0",
    "python": "3.12.12",
    "code_sha256": "53358bc8ab0704f5b1e76b0c3fa958cb419f95231e8ef31bc1aaff91eca1c6cb"
  },
  "summary": {
    "200": {
      "level_means_over_seeds": {
        "none": 2.7410566735267636,
        "flat": 2.7433232617378236,
        "top_only": 2.74224155664444,
        "top_mid": 2.74345914999644,
        "hierarchy": 2.7451548020044965
      },
      "successive_differences_of_level_means": {
        "top_only_minus_none": 0.001184883117676172,
        "top_mid_minus_top_only": 0.0012175933520000015,
        "hierarchy_minus_top_mid": 0.0016956520080566584
      },
      "mean_eval_nll_labeled_over_seeds": {
        "none": 2.7410566735267636,
        "flat": 2.7433232617378236,
        "top_only": 2.74224155664444,
        "top_mid": 2.74345914999644,
        "hierarchy": 2.7451548020044965,
        "scrambled_1": 2.744917451540629,
        "scrambled_2": 2.7457801365852355,
        "scrambled_3": 2.7458112796147662,
        "scrambled_4": 2.7454263122876483,
        "scrambled_5": 2.7454017345110575,
        "scrambled_mean": 2.7454673829078673
      },
      "paired_diff_hierarchy_minus_scrambled_mean": {
        "mean": -0.0003125809033710958,
        "min": -0.0013634244600928547,
        "max": 0.0002701663970947621,
        "per_seed": {
          "2027": 0.0002701663970947621,
          "3037": -0.0003119405110676432,
          "4049": 0.00026171127955132434,
          "5051": -0.0013634244600928547,
          "6067": -0.0004194172223410675
        }
      },
      "n_scrambled_better_than_hierarchy": {
        "mean_over_seeds": 2.0,
        "per_seed": {
          "2027": 3,
          "3037": 1,
          "4049": 4,
          "5051": 0,
          "6067": 2
        }
      },
      "hierarchy_rank_among_6_codes": {
        "mean_over_seeds": 3.0,
        "per_seed": {
          "2027": 4,
          "3037": 2,
          "4049": 5,
          "5051": 1,
          "6067": 3
        }
      },
      "between_permutation_range_of_scrambled": {
        "mean_over_seeds": 0.0016746497154235484,
        "per_seed": {
          "2027": 0.0015022635459898126,
          "3037": 0.000884536902109545,
          "4049": 0.002204402287801166,
          "5051": 0.0011124412218732616,
          "6067": 0.002669604619343957
        }
      },
      "seeds_where_hierarchy_beats_scrambled_mean": 3
    },
    "400": {
      "level_means_over_seeds": {
        "none": 2.620016299088796,
        "flat": 2.62081573565801,
        "top_only": 2.6203642344474796,
        "top_mid": 2.619529194831848,
        "hierarchy": 2.621239082813263
      },
      "successive_differences_of_level_means": {
        "top_only_minus_none": 0.00034793535868349323,
        "top_mid_minus_top_only": -0.0008350396156315831,
        "hierarchy_minus_top_mid": 0.0017098879814150258
      },
      "mean_eval_nll_labeled_over_seeds": {
        "none": 2.620016299088796,
        "flat": 2.62081573565801,
        "top_only": 2.6203642344474796,
        "top_mid": 2.619529194831848,
        "hierarchy": 2.621239082813263,
        "scrambled_1": 2.6216838773091635,
        "scrambled_2": 2.6215205423037213,
        "scrambled_3": 2.621815048058828,
        "scrambled_4": 2.621948627630869,
        "scrambled_5": 2.6219596028327943,
        "scrambled_mean": 2.621785539627075
      },
      "paired_diff_hierarchy_minus_scrambled_mean": {
        "mean": -0.000546456813812135,
        "min": -0.0018284726142883123,
        "max": 8.851369222018235e-05,
        "per_seed": {
          "2027": -0.00046612501144416285,
          "3037": -0.00012386480967174762,
          "4049": -0.0018284726142883123,
          "5051": -0.0004023353258766349,
          "6067": 8.851369222018235e-05
        }
      },
      "n_scrambled_better_than_hierarchy": {
        "mean_over_seeds": 1.6,
        "per_seed": {
          "2027": 3,
          "3037": 1,
          "4049": 0,
          "5051": 1,
          "6067": 3
        }
      },
      "hierarchy_rank_among_6_codes": {
        "mean_over_seeds": 2.6,
        "per_seed": {
          "2027": 4,
          "3037": 2,
          "4049": 1,
          "5051": 2,
          "6067": 4
        }
      },
      "between_permutation_range_of_scrambled": {
        "mean_over_seeds": 0.001176547209421752,
        "per_seed": {
          "2027": 0.0018871784210205966,
          "3037": 0.0007346391677853781,
          "4049": 0.0008720755577087402,
          "5051": 0.0009383122126260801,
          "6067": 0.0014505306879679658
        }
      },
      "seeds_where_hierarchy_beats_scrambled_mean": 4
    },
    "800": {
      "level_means_over_seeds": {
        "none": 2.515298938751221,
        "flat": 2.5216744407018026,
        "top_only": 2.519926375548045,
        "top_mid": 2.518661022981008,
        "hierarchy": 2.5198265504837036
      },
      "successive_differences_of_level_means": {
        "top_only_minus_none": 0.0046274367968242025,
        "top_mid_minus_top_only": -0.0012653525670369703,
        "hierarchy_minus_top_mid": 0.001165527502695607
      },
      "mean_eval_nll_labeled_over_seeds": {
        "none": 2.515298938751221,
        "flat": 2.5216744407018026,
        "top_only": 2.519926375548045,
        "top_mid": 2.518661022981008,
        "hierarchy": 2.5198265504837036,
        "scrambled_1": 2.520171813170115,
        "scrambled_2": 2.521135621070862,
        "scrambled_3": 2.5200824507077533,
        "scrambled_4": 2.5207991274197896,
        "scrambled_5": 2.520606079896291,
        "scrambled_mean": 2.5205590184529623
      },
      "paired_diff_hierarchy_minus_scrambled_mean": {
        "mean": -0.0007324679692586322,
        "min": -0.0017726922035219417,
        "max": 0.0002179431915281249,
        "per_seed": {
          "2027": -0.0017726922035219417,
          "3037": -0.00014568487803146013,
          "4049": -0.0010885071754453968,
          "5051": 0.0002179431915281249,
          "6067": -0.0008733987808224875
        }
      },
      "n_scrambled_better_than_hierarchy": {
        "mean_over_seeds": 1.8,
        "per_seed": {
          "2027": 0,
          "3037": 3,
          "4049": 1,
          "5051": 3,
          "6067": 2
        }
      },
      "hierarchy_rank_among_6_codes": {
        "mean_over_seeds": 2.8,
        "per_seed": {
          "2027": 1,
          "3037": 4,
          "4049": 2,
          "5051": 4,
          "6067": 3
        }
      },
      "between_permutation_range_of_scrambled": {
        "mean_over_seeds": 0.0050846457481384276,
        "per_seed": {
          "2027": 0.0018771608670551387,
          "3037": 0.003037802378336618,
          "4049": 0.0070454875628152,
          "5051": 0.009087252616882147,
          "6067": 0.004375525315603035
        }
      },
      "seeds_where_hierarchy_beats_scrambled_mean": 4
    }
  },
  "elapsed_seconds": 165.2161654829997,
  "runs": [
    {
      "seed": 2027,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.741122762362162,
      "eval_nll_ablated": 2.741122762362162,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.504383065374226,
      "perplexity_per_byte_ablated": 15.504383065374226
    },
    {
      "seed": 2027,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.622784169514974,
      "eval_nll_ablated": 2.622784169514974,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.774019454307584,
      "perplexity_per_byte_ablated": 13.774019454307584
    },
    {
      "seed": 2027,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5312110781669617,
      "eval_nll_ablated": 2.5312110781669617,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.568718623765305,
      "perplexity_per_byte_ablated": 12.568718623765305
    },
    {
      "seed": 2027,
      "condition": "flat",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7460975964864094,
      "eval_nll_ablated": 2.7494224270184833,
      "label_use_gap": 0.003324830532073886,
      "perplexity_per_byte_labeled": 15.581706976665613,
      "perplexity_per_byte_ablated": 15.633599731266886
    },
    {
      "seed": 2027,
      "condition": "flat",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.623246208826701,
      "eval_nll_ablated": 2.6265785336494445,
      "label_use_gap": 0.003332324822743704,
      "perplexity_per_byte_labeled": 13.780385063243507,
      "perplexity_per_byte_ablated": 13.826382378890743
    },
    {
      "seed": 2027,
      "condition": "flat",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.529518977801005,
      "eval_nll_ablated": 2.537992775440216,
      "label_use_gap": 0.008473797639211078,
      "perplexity_per_byte_labeled": 12.547469073638347,
      "perplexity_per_byte_ablated": 12.654245549652638
    },
    {
      "seed": 2027,
      "condition": "top_only",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7456389904022216,
      "eval_nll_ablated": 2.7506255904833474,
      "label_use_gap": 0.004986600081125836,
      "perplexity_per_byte_labeled": 15.574562749362352,
      "perplexity_per_byte_ablated": 15.652420827445045
    },
    {
      "seed": 2027,
      "condition": "top_only",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.622648247083028,
      "eval_nll_ablated": 2.629657566547394,
      "label_use_gap": 0.007009319464365671,
      "perplexity_per_byte_labeled": 13.77214738331679,
      "perplexity_per_byte_ablated": 13.869019872532588
    },
    {
      "seed": 2027,
      "condition": "top_only",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5257646679878234,
      "eval_nll_ablated": 2.535215910275777,
      "label_use_gap": 0.009451242287953576,
      "perplexity_per_byte_labeled": 12.500450304105357,
      "perplexity_per_byte_ablated": 12.619155159193424
    },
    {
      "seed": 2027,
      "condition": "top_mid",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7467618942260743,
      "eval_nll_ablated": 2.7620529850323994,
      "label_use_gap": 0.01529109080632507,
      "perplexity_per_byte_labeled": 15.592061308189074,
      "perplexity_per_byte_ablated": 15.832313108319278
    },
    {
      "seed": 2027,
      "condition": "top_mid",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6213756958643595,
      "eval_nll_ablated": 2.6372761925061545,
      "label_use_gap": 0.015900496641795048,
      "perplexity_per_byte_labeled": 13.754632766867298,
      "perplexity_per_byte_ablated": 13.975086274439564
    },
    {
      "seed": 2027,
      "condition": "top_mid",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.522379199663798,
      "eval_nll_ablated": 2.5526976307233173,
      "label_use_gap": 0.03031843105951948,
      "perplexity_per_byte_labeled": 12.458201981096453,
      "perplexity_per_byte_ablated": 12.84169926123692
    },
    {
      "seed": 2027,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.755657680829366,
      "eval_nll_ablated": 2.7648485620816547,
      "label_use_gap": 0.00919088125228873,
      "perplexity_per_byte_labeled": 15.731383730790084,
      "perplexity_per_byte_ablated": 15.876635483923653
    },
    {
      "seed": 2027,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6224042534828187,
      "eval_nll_ablated": 2.6488504807154336,
      "label_use_gap": 0.026446227232614916,
      "perplexity_per_byte_labeled": 13.768787477408557,
      "perplexity_per_byte_ablated": 14.137777652951232
    },
    {
      "seed": 2027,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.515796907742818,
      "eval_nll_ablated": 2.554167918364207,
      "label_use_gap": 0.03837101062138881,
      "perplexity_per_byte_labeled": 12.376467753225317,
      "perplexity_per_byte_ablated": 12.860594139998272
    },
    {
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7544564525286357,
      "eval_nll_ablated": 2.7672189474105835,
      "label_use_gap": 0.012762494881947806,
      "perplexity_per_byte_labeled": 15.712498092695407,
      "perplexity_per_byte_ablated": 15.914313866247792
    },
    {
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.622199801603953,
      "eval_nll_ablated": 2.655825209617615,
      "label_use_gap": 0.03362540801366176,
      "perplexity_per_byte_labeled": 13.765972710691077,
      "perplexity_per_byte_ablated": 14.236729499370536
    },
    {
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5187567194302876,
      "eval_nll_ablated": 2.5793508847554523,
      "label_use_gap": 0.060594165325164706,
      "perplexity_per_byte_labeled": 12.413154032587652,
      "perplexity_per_byte_ablated": 13.188574475804083
    },
    {
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7559587160746255,
      "eval_nll_ablated": 2.7676771759986876,
      "label_use_gap": 0.011718459924062063,
      "perplexity_per_byte_labeled": 15.736120144627735,
      "perplexity_per_byte_ablated": 15.921607930868204
    },
    {
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6221398591995237,
      "eval_nll_ablated": 2.6517008980115255,
      "label_use_gap": 0.029561038812001783,
      "perplexity_per_byte_labeled": 13.765147569918197,
      "perplexity_per_byte_ablated": 14.178133707375274
    },
    {
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5179242650667826,
      "eval_nll_ablated": 2.5732008775075275,
      "label_use_gap": 0.05527661244074489,
      "perplexity_per_byte_labeled": 12.40282494819054,
      "perplexity_per_byte_ablated": 13.107713549692892
    },
    {
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7554256836573283,
      "eval_nll_ablated": 2.769458023707072,
      "label_use_gap": 0.014032340049743741,
      "perplexity_per_byte_labeled": 15.727734517572241,
      "perplexity_per_byte_ablated": 15.94998715190086
    },
    {
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6223039905230205,
      "eval_nll_ablated": 2.6534973343213397,
      "label_use_gap": 0.031193343798319173,
      "perplexity_per_byte_labeled": 13.767407047227405,
      "perplexity_per_byte_ablated": 14.203626712997192
    },
    {
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.517093098163605,
      "eval_nll_ablated": 2.574970543384552,
      "label_use_gap": 0.05787744522094718,
      "perplexity_per_byte_labeled": 12.39252041357499,
      "perplexity_per_byte_ablated": 13.130930360027067
    },
    {
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.755787924925486,
      "eval_nll_ablated": 2.766776132583618,
      "label_use_gap": 0.010988207658131799,
      "perplexity_per_byte_labeled": 15.73343278408047,
      "perplexity_per_byte_ablated": 15.907268332155441
    },
    {
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6240270376205443,
      "eval_nll_ablated": 2.650338300069173,
      "label_use_gap": 0.026311262448628803,
      "perplexity_per_byte_labeled": 13.791149386691075,
      "perplexity_per_byte_ablated": 14.158827767663139
    },
    {
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5168795585632324,
      "eval_nll_ablated": 2.557055926322937,
      "label_use_gap": 0.04017636775970468,
      "perplexity_per_byte_labeled": 12.389874402242425,
      "perplexity_per_byte_ablated": 12.897789322367462
    },
    {
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7553087949752806,
      "eval_nll_ablated": 2.7657041192054748,
      "label_use_gap": 0.01039532423019418,
      "perplexity_per_byte_labeled": 15.725896230852435,
      "perplexity_per_byte_ablated": 15.890224664845574
    },
    {
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6236812035242716,
      "eval_nll_ablated": 2.650618481636047,
      "label_use_gap": 0.026937278111775598,
      "perplexity_per_byte_labeled": 13.786380761630463,
      "perplexity_per_byte_ablated": 14.162795366010187
    },
    {
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5171943585077923,
      "eval_nll_ablated": 2.5627002318700156,
      "label_use_gap": 0.04550587336222334,
      "perplexity_per_byte_labeled": 12.393775347993893,
      "perplexity_per_byte_ablated": 12.970794223351676
    },
    {
      "seed": 3037,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7360187570254006,
      "eval_nll_ablated": 2.7360187570254006,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.425450219653968,
      "perplexity_per_byte_ablated": 15.425450219653968
    },
    {
      "seed": 3037,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6173327445983885,
      "eval_nll_ablated": 2.6173327445983885,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.699135718434324,
      "perplexity_per_byte_ablated": 13.699135718434324
    },
    {
      "seed": 3037,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4984604636828105,
      "eval_nll_ablated": 2.4984604636828105,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.163752998714438,
      "perplexity_per_byte_ablated": 12.163752998714438
    },
    {
      "seed": 3037,
      "condition": "flat",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.740092162291209,
      "eval_nll_ablated": 2.7459948460261026,
      "label_use_gap": 0.0059026837348934436,
      "perplexity_per_byte_labeled": 15.488412478144713,
      "perplexity_per_byte_ablated": 15.580106031351738
    },
    {
      "seed": 3037,
      "condition": "flat",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6223777294158936,
      "eval_nll_ablated": 2.6284648537635804,
      "label_use_gap": 0.006087124347686768,
      "perplexity_per_byte_labeled": 13.768422278011338,
      "perplexity_per_byte_ablated": 13.852487976093576
    },
    {
      "seed": 3037,
      "condition": "flat",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5167282025019326,
      "eval_nll_ablated": 2.5245769103368123,
      "label_use_gap": 0.007848707834879676,
      "perplexity_per_byte_labeled": 12.387999261563436,
      "perplexity_per_byte_ablated": 12.485611612744664
    },
    {
      "seed": 3037,
      "condition": "top_only",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.738525366783142,
      "eval_nll_ablated": 2.7501901825269064,
      "label_use_gap": 0.011664815743764212,
      "perplexity_per_byte_labeled": 15.46416430397252,
      "perplexity_per_byte_ablated": 15.645607122357562
    },
    {
      "seed": 3037,
      "condition": "top_only",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6226794838905336,
      "eval_nll_ablated": 2.632752792040507,
      "label_use_gap": 0.010073308149973315,
      "perplexity_per_byte_labeled": 13.772577587952615,
      "perplexity_per_byte_ablated": 13.912014120527527
    },
    {
      "seed": 3037,
      "condition": "top_only",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.513990835348765,
      "eval_nll_ablated": 2.5253035187721253,
      "label_use_gap": 0.011312683423360337,
      "perplexity_per_byte_labeled": 12.354135129718289,
      "perplexity_per_byte_ablated": 12.49468706021158
    },
    {
      "seed": 3037,
      "condition": "top_mid",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7383634289105734,
      "eval_nll_ablated": 2.750613776842753,
      "label_use_gap": 0.012250347932179828,
      "perplexity_per_byte_labeled": 15.461660272858287,
      "perplexity_per_byte_ablated": 15.6522359164632
    },
    {
      "seed": 3037,
      "condition": "top_mid",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.620990490913391,
      "eval_nll_ablated": 2.6358664870262145,
      "label_use_gap": 0.014875996112823309,
      "perplexity_per_byte_labeled": 13.749335434571567,
      "perplexity_per_byte_ablated": 13.955399398344671
    },
    {
      "seed": 3037,
      "condition": "top_mid",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5149285872777303,
      "eval_nll_ablated": 2.529041310151418,
      "label_use_gap": 0.014112722873687922,
      "perplexity_per_byte_labeled": 12.365725677446752,
      "perplexity_per_byte_ablated": 12.541476984805106
    },
    {
      "seed": 3037,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7384955128033956,
      "eval_nll_ablated": 2.7513768871625266,
      "label_use_gap": 0.012881374359130948,
      "perplexity_per_byte_labeled": 15.46370264401582,
      "perplexity_per_byte_ablated": 15.664184857818949
    },
    {
      "seed": 3037,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6233902017275494,
      "eval_nll_ablated": 2.639358711242676,
      "label_use_gap": 0.015968509515126517,
      "perplexity_per_byte_labeled": 13.782369483731378,
      "perplexity_per_byte_ablated": 14.004219978663919
    },
    {
      "seed": 3037,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5105364402135213,
      "eval_nll_ablated": 2.5448630650838218,
      "label_use_gap": 0.03432662487030047,
      "perplexity_per_byte_labeled": 12.311532690619254,
      "perplexity_per_byte_ablated": 12.741483212665491
    },
    {
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.739237570762634,
      "eval_nll_ablated": 2.752917202313741,
      "label_use_gap": 0.013679631551106741,
      "perplexity_per_byte_labeled": 15.475181866239465,
      "perplexity_per_byte_ablated": 15.688331230824883
    },
    {
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6236424406369525,
      "eval_nll_ablated": 2.641030768553416,
      "label_use_gap": 0.017388327916463275,
      "perplexity_per_byte_labeled": 13.78584637206377,
      "perplexity_per_byte_ablated": 14.02765542430454
    },
    {
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5099976499875387,
      "eval_nll_ablated": 2.538249742984772,
      "label_use_gap": 0.02825209299723319,
      "perplexity_per_byte_labeled": 12.304901143805411,
      "perplexity_per_byte_ablated": 12.657497697890117
    },
    {
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7387609402338664,
      "eval_nll_ablated": 2.751416389147441,
      "label_use_gap": 0.012655448913574396,
      "perplexity_per_byte_labeled": 15.467807679644613,
      "perplexity_per_byte_ablated": 15.664803636434307
    },
    {
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6230279326438906,
      "eval_nll_ablated": 2.6388460556666056,
      "label_use_gap": 0.015818123022715014,
      "perplexity_per_byte_labeled": 13.777377461650087,
      "perplexity_per_byte_ablated": 13.997042477153624
    },
    {
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5101327737172445,
      "eval_nll_ablated": 2.5419180671374004,
      "label_use_gap": 0.0317852934201559,
      "perplexity_per_byte_labeled": 12.306563940280725,
      "perplexity_per_byte_ablated": 12.704014770093398
    },
    {
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.738762072722117,
      "eval_nll_ablated": 2.7515345176060992,
      "label_use_gap": 0.012772444883982104,
      "perplexity_per_byte_labeled": 15.467825196764993,
      "perplexity_per_byte_ablated": 15.666654204843312
    },
    {
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6236024220784504,
      "eval_nll_ablated": 2.639956005414327,
      "label_use_gap": 0.016353583335876554,
      "perplexity_per_byte_labeled": 13.785294693402994,
      "perplexity_per_byte_ablated": 14.012587116208183
    },
    {
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.511931312084198,
      "eval_nll_ablated": 2.546951492627462,
      "label_use_gap": 0.035020180543263724,
      "perplexity_per_byte_labeled": 12.328717683899823,
      "perplexity_per_byte_ablated": 12.768120682685016
    },
    {
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.738923648993174,
      "eval_nll_ablated": 2.751608701546987,
      "label_use_gap": 0.012685052553812692,
      "perplexity_per_byte_labeled": 15.470324632200938,
      "perplexity_per_byte_ablated": 15.667816462102628
    },
    {
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.623762571811676,
      "eval_nll_ablated": 2.6406047264734904,
      "label_use_gap": 0.01684215466181449,
      "perplexity_per_byte_labeled": 13.787502581462201,
      "perplexity_per_byte_ablated": 14.021680325723185
    },
    {
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5121933460235595,
      "eval_nll_ablated": 2.54321786959966,
      "label_use_gap": 0.031024523576100638,
      "perplexity_per_byte_labeled": 12.331948649654663,
      "perplexity_per_byte_ablated": 12.720538216053676
    },
    {
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7383530338605246,
      "eval_nll_ablated": 2.750073758761088,
      "label_use_gap": 0.011720724900563528,
      "perplexity_per_byte_labeled": 15.461499548961283,
      "perplexity_per_byte_ablated": 15.643785707887888
    },
    {
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6235349655151365,
      "eval_nll_ablated": 2.6385919491449994,
      "label_use_gap": 0.015056983629862852,
      "perplexity_per_byte_labeled": 13.784364816162222,
      "perplexity_per_byte_ablated": 13.993486189234092
    },
    {
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.509155543645223,
      "eval_nll_ablated": 2.5367073734601338,
      "label_use_gap": 0.02755182981491089,
      "perplexity_per_byte_labeled": 12.294543470254027,
      "perplexity_per_byte_ablated": 12.637990206929697
    },
    {
      "seed": 4049,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.74272407690684,
      "eval_nll_ablated": 2.74272407690684,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.529230348331396,
      "perplexity_per_byte_ablated": 15.529230348331396
    },
    {
      "seed": 4049,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6174652139345804,
      "eval_nll_ablated": 2.6174652139345804,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.700950554051728,
      "perplexity_per_byte_ablated": 13.700950554051728
    },
    {
      "seed": 4049,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.486494727929433,
      "eval_nll_ablated": 2.486494727929433,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.019072077663717,
      "perplexity_per_byte_ablated": 12.019072077663717
    },
    {
      "seed": 4049,
      "condition": "flat",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.743322467803955,
      "eval_nll_ablated": 2.749962401390076,
      "label_use_gap": 0.006639933586120694,
      "perplexity_per_byte_labeled": 15.538525679254382,
      "perplexity_per_byte_ablated": 15.64204375403028
    },
    {
      "seed": 4049,
      "condition": "flat",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.619122616449992,
      "eval_nll_ablated": 2.624290072917938,
      "label_use_gap": 0.005167456467946252,
      "perplexity_per_byte_labeled": 13.723677372504062,
      "perplexity_per_byte_ablated": 13.794777422901278
    },
    {
      "seed": 4049,
      "condition": "flat",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5017425139745075,
      "eval_nll_ablated": 2.508978072802226,
      "label_use_gap": 0.007235558827718425,
      "perplexity_per_byte_labeled": 12.203740632609671,
      "perplexity_per_byte_ablated": 12.292361740862503
    },
    {
      "seed": 4049,
      "condition": "top_only",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.741611949602763,
      "eval_nll_ablated": 2.754396140575409,
      "label_use_gap": 0.012784190972646048,
      "perplexity_per_byte_labeled": 15.511969467177387,
      "perplexity_per_byte_ablated": 15.711550469822145
    },
    {
      "seed": 4049,
      "condition": "top_only",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.616963970661163,
      "eval_nll_ablated": 2.6297266681989035,
      "label_use_gap": 0.012762697537740397,
      "perplexity_per_byte_labeled": 13.694084765605973,
      "perplexity_per_byte_ablated": 13.869978277823915
    },
    {
      "seed": 4049,
      "condition": "top_only",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.502438978354136,
      "eval_nll_ablated": 2.520884641011556,
      "label_use_gap": 0.01844566265741987,
      "perplexity_per_byte_labeled": 12.212243063735034,
      "perplexity_per_byte_ablated": 12.439596374582612
    },
    {
      "seed": 4049,
      "condition": "top_mid",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7426485260327658,
      "eval_nll_ablated": 2.7594103813171387,
      "label_use_gap": 0.016761855284372906,
      "perplexity_per_byte_labeled": 15.52805714572368,
      "perplexity_per_byte_ablated": 15.79052981154134
    },
    {
      "seed": 4049,
      "condition": "top_mid",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6155675570170085,
      "eval_nll_ablated": 2.6290356079737345,
      "label_use_gap": 0.013468050956725985,
      "perplexity_per_byte_labeled": 13.674975504117016,
      "perplexity_per_byte_ablated": 13.860396598652157
    },
    {
      "seed": 4049,
      "condition": "top_mid",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5053526282310488,
      "eval_nll_ablated": 2.5234015623728436,
      "label_use_gap": 0.01804893414179487,
      "perplexity_per_byte_labeled": 12.247877151653253,
      "perplexity_per_byte_ablated": 12.470945295258309
    },
    {
      "seed": 4049,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7438509305318197,
      "eval_nll_ablated": 2.757043914000193,
      "label_use_gap": 0.013192983468373498,
      "perplexity_per_byte_labeled": 15.546739381048306,
      "perplexity_per_byte_ablated": 15.753206218774489
    },
    {
      "seed": 4049,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.615667696793874,
      "eval_nll_ablated": 2.6314353307088214,
      "label_use_gap": 0.01576763391494751,
      "perplexity_per_byte_labeled": 13.676344981681083,
      "perplexity_per_byte_ablated": 13.893697648149246
    },
    {
      "seed": 4049,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.500712815920512,
      "eval_nll_ablated": 2.5475839734077455,
      "label_use_gap": 0.04687115748723336,
      "perplexity_per_byte_labeled": 12.191180932088132,
      "perplexity_per_byte_ablated": 12.776198827983682
    },
    {
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7422760009765623,
      "eval_nll_ablated": 2.7567723949750262,
      "label_use_gap": 0.014496393998463919,
      "perplexity_per_byte_labeled": 15.522273632681383,
      "perplexity_per_byte_ablated": 15.748929504209672
    },
    {
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6173415621121725,
      "eval_nll_ablated": 2.6390868067741393,
      "label_use_gap": 0.02174524466196681,
      "perplexity_per_byte_labeled": 13.699256511284895,
      "perplexity_per_byte_ablated": 14.00041268630671
    },
    {
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5027395844459535,
      "eval_nll_ablated": 2.5538566788037618,
      "label_use_gap": 0.05111709435780831,
      "perplexity_per_byte_labeled": 12.215914690223743,
      "perplexity_per_byte_ablated": 12.856592037169744
    },
    {
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7436550736427305,
      "eval_nll_ablated": 2.755983579158783,
      "label_use_gap": 0.012328505516052246,
      "perplexity_per_byte_labeled": 15.543694743204036,
      "perplexity_per_byte_ablated": 15.736511397971062
    },
    {
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.617518353462219,
      "eval_nll_ablated": 2.637151535352071,
      "label_use_gap": 0.01963318188985186,
      "perplexity_per_byte_labeled": 13.70167863543715,
      "perplexity_per_byte_ablated": 13.973344288534278
    },
    {
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.496725920836131,
      "eval_nll_ablated": 2.5404717961947125,
      "label_use_gap": 0.04374587535858154,
      "perplexity_per_byte_labeled": 12.142672735556816,
      "perplexity_per_byte_ablated": 12.685654602764519
    },
    {
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7444804032643635,
      "eval_nll_ablated": 2.7573309381802877,
      "label_use_gap": 0.01285053491592425,
      "perplexity_per_byte_labeled": 15.556528710303319,
      "perplexity_per_byte_ablated": 15.757728418832627
    },
    {
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6177183389663696,
      "eval_nll_ablated": 2.635655411084493,
      "label_use_gap": 0.017937072118123343,
      "perplexity_per_byte_labeled": 13.704419046558883,
      "perplexity_per_byte_ablated": 13.952454060130519
    },
    {
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5029831528663635,
      "eval_nll_ablated": 2.5504605889320375,
      "label_use_gap": 0.047477436065674006,
      "perplexity_per_byte_labeled": 12.21889046365621,
      "perplexity_per_byte_ablated": 12.813003951588728
    },
    {
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7438379287719727,
      "eval_nll_ablated": 2.756937511761983,
      "label_use_gap": 0.013099582990010461,
      "perplexity_per_byte_labeled": 15.546537247390521,
      "perplexity_per_byte_ablated": 15.751530131545124
    },
    {
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.617015258471171,
      "eval_nll_ablated": 2.6352463165918985,
      "label_use_gap": 0.01823105812072745,
      "perplexity_per_byte_labeled": 13.694787123234702,
      "perplexity_per_byte_ablated": 13.94674735538667
    },
    {
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.503771408398946,
      "eval_nll_ablated": 2.558566363652547,
      "label_use_gap": 0.0547949552536009,
      "perplexity_per_byte_labeled": 12.228525868747985,
      "perplexity_per_byte_ablated": 12.91728534488295
    },
    {
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.743696689605713,
      "eval_nll_ablated": 2.7578200260798136,
      "label_use_gap": 0.01412333647410069,
      "perplexity_per_byte_labeled": 15.544341622489236,
      "perplexity_per_byte_ablated": 15.76543721811285
    },
    {
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6178873340288797,
      "eval_nll_ablated": 2.637756586074829,
      "label_use_gap": 0.01986925204594936,
      "perplexity_per_byte_labeled": 13.706735221417867,
      "perplexity_per_byte_ablated": 13.981801428836828
    },
    {
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5027865489323933,
      "eval_nll_ablated": 2.5554967482884723,
      "label_use_gap": 0.05271019935607901,
      "perplexity_per_byte_labeled": 12.216488417855869,
      "perplexity_per_byte_ablated": 12.877695041914048
    },
    {
      "seed": 5051,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.733423086007436,
      "eval_nll_ablated": 2.733423086007436,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.385462745195346,
      "perplexity_per_byte_ablated": 15.385462745195346
    },
    {
      "seed": 5051,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.619443726539612,
      "eval_nll_ablated": 2.619443726539612,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.728084891386583,
      "perplexity_per_byte_ablated": 13.728084891386583
    },
    {
      "seed": 5051,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.527475611368815,
      "eval_nll_ablated": 2.527475611368815,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.521856173706203,
      "perplexity_per_byte_ablated": 12.521856173706203
    },
    {
      "seed": 5051,
      "condition": "flat",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.73391398191452,
      "eval_nll_ablated": 2.734453999996185,
      "label_use_gap": 0.0005400180816650391,
      "perplexity_per_byte_labeled": 15.393017259974044,
      "perplexity_per_byte_ablated": 15.4013320124821
    },
    {
      "seed": 5051,
      "condition": "flat",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.619692413012187,
      "eval_nll_ablated": 2.619947437445323,
      "label_use_gap": 0.00025502443313607515,
      "perplexity_per_byte_labeled": 13.731499304935063,
      "perplexity_per_byte_ablated": 13.735001619330303
    },
    {
      "seed": 5051,
      "condition": "flat",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.536522750059764,
      "eval_nll_ablated": 2.537625018755595,
      "label_use_gap": 0.0011022686958312988,
      "perplexity_per_byte_labeled": 12.635657153578599,
      "perplexity_per_byte_ablated": 12.649592721869617
    },
    {
      "seed": 5051,
      "condition": "top_only",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7327037533124288,
      "eval_nll_ablated": 2.7344650586446124,
      "label_use_gap": 0.0017613053321836603,
      "perplexity_per_byte_labeled": 15.37439945838392,
      "perplexity_per_byte_ablated": 15.401502331339884
    },
    {
      "seed": 5051,
      "condition": "top_only",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6186869502067567,
      "eval_nll_ablated": 2.620968759059906,
      "label_use_gap": 0.0022818088531493252,
      "perplexity_per_byte_labeled": 13.717699731762503,
      "perplexity_per_byte_ablated": 13.749036639275083
    },
    {
      "seed": 5051,
      "condition": "top_only",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.535843050479889,
      "eval_nll_ablated": 2.5372100353240965,
      "label_use_gap": 0.0013669848442074972,
      "perplexity_per_byte_labeled": 12.627071620841903,
      "perplexity_per_byte_ablated": 12.64434443952526
    },
    {
      "seed": 5051,
      "condition": "top_mid",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7378147721290587,
      "eval_nll_ablated": 2.7724106152852377,
      "label_use_gap": 0.03459584315617903,
      "perplexity_per_byte_labeled": 15.453179454838011,
      "perplexity_per_byte_ablated": 15.997150542488935
    },
    {
      "seed": 5051,
      "condition": "top_mid",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6191750605901083,
      "eval_nll_ablated": 2.6568966110547385,
      "label_use_gap": 0.037721550464630216,
      "perplexity_per_byte_labeled": 13.724397117836144,
      "perplexity_per_byte_ablated": 14.251990925913383
    },
    {
      "seed": 5051,
      "condition": "top_mid",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.522354165712992,
      "eval_nll_ablated": 2.582733126481374,
      "label_use_gap": 0.060378960768381784,
      "perplexity_per_byte_labeled": 12.457890106984667,
      "perplexity_per_byte_ablated": 13.233256943559367
    },
    {
      "seed": 5051,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7380093256632487,
      "eval_nll_ablated": 2.8241296847661337,
      "label_use_gap": 0.08612035910288496,
      "perplexity_per_byte_labeled": 15.456186217994139,
      "perplexity_per_byte_ablated": 16.84627703260779
    },
    {
      "seed": 5051,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6249491771062217,
      "eval_nll_ablated": 2.7359302560488383,
      "label_use_gap": 0.11098107894261666,
      "perplexity_per_byte_labeled": 13.803872615487656,
      "perplexity_per_byte_ablated": 15.424085112653156
    },
    {
      "seed": 5051,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5464496850967406,
      "eval_nll_ablated": 2.6548113544782,
      "label_use_gap": 0.10836166938145952,
      "perplexity_per_byte_labeled": 12.76171515087975,
      "perplexity_per_byte_ablated": 14.22230283251008
    },
    {
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.739382592837016,
      "eval_nll_ablated": 2.8237810929616294,
      "label_use_gap": 0.08439850012461347,
      "perplexity_per_byte_labeled": 15.477426271955395,
      "perplexity_per_byte_ablated": 16.840405581927065
    },
    {
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.625090972582499,
      "eval_nll_ablated": 2.7423828204472858,
      "label_use_gap": 0.11729184786478664,
      "perplexity_per_byte_labeled": 13.805830080956238,
      "perplexity_per_byte_ablated": 15.523931802296074
    },
    {
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.544010591506958,
      "eval_nll_ablated": 2.6708620389302573,
      "label_use_gap": 0.12685144742329912,
      "perplexity_per_byte_labeled": 12.730626063270632,
      "perplexity_per_byte_ablated": 14.452422375191631
    },
    {
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.739650229612986,
      "eval_nll_ablated": 2.823433438936869,
      "label_use_gap": 0.08378320932388306,
      "perplexity_per_byte_labeled": 15.48156915479231,
      "perplexity_per_byte_ablated": 16.83455196472366
    },
    {
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.625513168176015,
      "eval_nll_ablated": 2.7345850586891176,
      "label_use_gap": 0.10907189051310251,
      "perplexity_per_byte_labeled": 13.811660072192952,
      "perplexity_per_byte_ablated": 15.403350623200426
    },
    {
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5513836900393168,
      "eval_nll_ablated": 2.671517868836721,
      "label_use_gap": 0.12013417879740418,
      "perplexity_per_byte_labeled": 12.824837110484784,
      "perplexity_per_byte_ablated": 14.461903814772102
    },
    {
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7386183540026345,
      "eval_nll_ablated": 2.8176608721415204,
      "label_use_gap": 0.07904251813888585,
      "perplexity_per_byte_labeled": 15.46560234047139,
      "perplexity_per_byte_ablated": 16.73765333501915
    },
    {
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.62485507329305,
      "eval_nll_ablated": 2.7378658095995587,
      "label_use_gap": 0.11301073630650871,
      "perplexity_per_byte_labeled": 13.802573679556373,
      "perplexity_per_byte_ablated": 15.453968166155313
    },
    {
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5422964374224346,
      "eval_nll_ablated": 2.664745235443115,
      "label_use_gap": 0.12244879802068054,
      "perplexity_per_byte_labeled": 12.708822501276988,
      "perplexity_per_byte_ablated": 14.364289567933461
    },
    {
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.739481778939565,
      "eval_nll_ablated": 2.826668938000997,
      "label_use_gap": 0.08718715906143215,
      "perplexity_per_byte_labeled": 15.47896149367987,
      "perplexity_per_byte_ablated": 16.88910835282845
    },
    {
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.625793385505676,
      "eval_nll_ablated": 2.7412965138753256,
      "label_use_gap": 0.11550312836964949,
      "perplexity_per_byte_labeled": 13.815530881005094,
      "perplexity_per_byte_ablated": 15.507077209441526
    },
    {
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5460153937339784,
      "eval_nll_ablated": 2.6714833974838257,
      "label_use_gap": 0.12546800374984723,
      "perplexity_per_byte_labeled": 12.756174051528577,
      "perplexity_per_byte_ablated": 14.46140530197442
    },
    {
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.739730795224508,
      "eval_nll_ablated": 2.822732170422872,
      "label_use_gap": 0.08300137519836426,
      "perplexity_per_byte_labeled": 15.482816487123953,
      "perplexity_per_byte_ablated": 16.822750561943284
    },
    {
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.625504962603251,
      "eval_nll_ablated": 2.7329604466756185,
      "label_use_gap": 0.10745548407236738,
      "perplexity_per_byte_labeled": 13.811546740076217,
      "perplexity_per_byte_ablated": 15.378346471252245
    },
    {
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5474525968233745,
      "eval_nll_ablated": 2.6647233804066977,
      "label_use_gap": 0.1172707835833231,
      "perplexity_per_byte_labeled": 12.774520444872952,
      "perplexity_per_byte_ablated": 14.363975639292317
    },
    {
      "seed": 6067,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7519946853319803,
      "eval_nll_ablated": 2.7519946853319803,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.673865152476361,
      "perplexity_per_byte_ablated": 15.673865152476361
    },
    {
      "seed": 6067,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.623055640856425,
      "eval_nll_ablated": 2.623055640856425,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.777759213441769,
      "perplexity_per_byte_ablated": 13.777759213441769
    },
    {
      "seed": 6067,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.532852812608083,
      "eval_nll_ablated": 2.532852812608083,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.589370069467066,
      "perplexity_per_byte_ablated": 12.589370069467066
    },
    {
      "seed": 6067,
      "condition": "flat",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7531901001930237,
      "eval_nll_ablated": 2.754967037836711,
      "label_use_gap": 0.0017769376436871376,
      "perplexity_per_byte_labeled": 15.69261312738093,
      "perplexity_per_byte_ablated": 15.720522711826678
    },
    {
      "seed": 6067,
      "condition": "flat",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6196397105852762,
      "eval_nll_ablated": 2.6223485589027407,
      "label_use_gap": 0.0027088483174644296,
      "perplexity_per_byte_labeled": 13.730775640666172,
      "perplexity_per_byte_ablated": 13.76802065192603
    },
    {
      "seed": 6067,
      "condition": "flat",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.523859759171804,
      "eval_nll_ablated": 2.5307977040608725,
      "label_use_gap": 0.006937944889068515,
      "perplexity_per_byte_labeled": 12.476660751774553,
      "perplexity_per_byte_ablated": 12.563524114651269
    },
    {
      "seed": 6067,
      "condition": "top_only",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.752727723121643,
      "eval_nll_ablated": 2.7559228777885436,
      "label_use_gap": 0.003195154666900457,
      "perplexity_per_byte_labeled": 15.685358900104209,
      "perplexity_per_byte_ablated": 15.735556199157653
    },
    {
      "seed": 6067,
      "condition": "top_only",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.620842520395915,
      "eval_nll_ablated": 2.624587682882945,
      "label_use_gap": 0.0037451624870299405,
      "perplexity_per_byte_labeled": 13.747301088807667,
      "perplexity_per_byte_ablated": 13.798883497101377
    },
    {
      "seed": 6067,
      "condition": "top_only",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5215943455696106,
      "eval_nll_ablated": 2.534035050868988,
      "label_use_gap": 0.012440705299377353,
      "perplexity_per_byte_labeled": 12.448427946362695,
      "perplexity_per_byte_ablated": 12.60426250591338
    },
    {
      "seed": 6067,
      "condition": "top_mid",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.751707128683726,
      "eval_nll_ablated": 2.760552493731181,
      "label_use_gap": 0.008845365047454923,
      "perplexity_per_byte_labeled": 15.669358676312568,
      "perplexity_per_byte_ablated": 15.808574674332801
    },
    {
      "seed": 6067,
      "condition": "top_mid",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6205371697743733,
      "eval_nll_ablated": 2.631419535477956,
      "label_use_gap": 0.010882365703582586,
      "perplexity_per_byte_labeled": 13.74310398270277,
      "perplexity_per_byte_ablated": 13.893478195720473
    },
    {
      "seed": 6067,
      "condition": "top_mid",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.52829053401947,
      "eval_nll_ablated": 2.5462010304133096,
      "label_use_gap": 0.01791049639383946,
      "perplexity_per_byte_labeled": 12.532064676935702,
      "perplexity_per_byte_ablated": 12.758542285129183
    },
    {
      "seed": 6067,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7497605601946513,
      "eval_nll_ablated": 2.760099971294403,
      "label_use_gap": 0.010339411099751672,
      "perplexity_per_byte_labeled": 15.638886863826047,
      "perplexity_per_byte_ablated": 15.801422557967797
    },
    {
      "seed": 6067,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6197840849558514,
      "eval_nll_ablated": 2.6413922866185504,
      "label_use_gap": 0.021608201662699056,
      "perplexity_per_byte_labeled": 13.732758155865548,
      "perplexity_per_byte_ablated": 14.032727591936782
    },
    {
      "seed": 6067,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.525636903444926,
      "eval_nll_ablated": 2.5528238654136657,
      "label_use_gap": 0.02718696196873971,
      "perplexity_per_byte_labeled": 12.498853291809185,
      "perplexity_per_byte_ablated": 12.843320431488518
    },
    {
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.749234640598297,
      "eval_nll_ablated": 2.759211039543152,
      "label_use_gap": 0.009976398944854914,
      "perplexity_per_byte_labeled": 15.630664229171067,
      "perplexity_per_byte_ablated": 15.787382413030942
    },
    {
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.62014460961024,
      "eval_nll_ablated": 2.635776424407959,
      "label_use_gap": 0.015631814797719024,
      "perplexity_per_byte_labeled": 13.737710046339158,
      "perplexity_per_byte_ablated": 13.954142595132396
    },
    {
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.525354520479838,
      "eval_nll_ablated": 2.548969312508901,
      "label_use_gap": 0.023614792029063025,
      "perplexity_per_byte_labeled": 12.495324326839691,
      "perplexity_per_byte_ablated": 12.79391046126066
    },
    {
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.750875723361969,
      "eval_nll_ablated": 2.761782721678416,
      "label_use_gap": 0.010906998316446703,
      "perplexity_per_byte_labeled": 15.6563365022231,
      "perplexity_per_byte_ablated": 15.828034792439347
    },
    {
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.619403398036957,
      "eval_nll_ablated": 2.6389211813608804,
      "label_use_gap": 0.01951778332392351,
      "perplexity_per_byte_labeled": 13.727531269442037,
      "perplexity_per_byte_ablated": 13.998094054187247
    },
    {
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5295114556948346,
      "eval_nll_ablated": 2.5525363246599833,
      "label_use_gap": 0.02302486896514866,
      "perplexity_per_byte_labeled": 12.547374690598785,
      "perplexity_per_byte_ablated": 12.839627984341822
    },
    {
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7517698844273886,
      "eval_nll_ablated": 2.7595772544542947,
      "label_use_gap": 0.007807370026906124,
      "perplexity_per_byte_labeled": 15.670342049424848,
      "perplexity_per_byte_ablated": 15.793165046657208
    },
    {
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.620595415433248,
      "eval_nll_ablated": 2.6383256117502847,
      "label_use_gap": 0.01773019631703665,
      "perplexity_per_byte_labeled": 13.74390448216181,
      "perplexity_per_byte_ablated": 13.98975969685364
    },
    {
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5261082530021666,
      "eval_nll_ablated": 2.5456446329752604,
      "label_use_gap": 0.019536379973093876,
      "perplexity_per_byte_labeled": 12.504746009430116,
      "perplexity_per_byte_ablated": 12.751445439404971
    },
    {
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7491002798080446,
      "eval_nll_ablated": 2.760260542233785,
      "label_use_gap": 0.01116026242574053,
      "perplexity_per_byte_labeled": 15.628564221855546,
      "perplexity_per_byte_ablated": 15.803960010946657
    },
    {
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.61914488474528,
      "eval_nll_ablated": 2.6392620762189227,
      "label_use_gap": 0.02011719147364266,
      "perplexity_per_byte_labeled": 13.723982978806886,
      "perplexity_per_byte_ablated": 14.002866745919528
    },
    {
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5251359303792316,
      "eval_nll_ablated": 2.5489692370096844,
      "label_use_gap": 0.0238333066304528,
      "perplexity_per_byte_labeled": 12.492593271139722,
      "perplexity_per_byte_ablated": 12.793909495330478
    },
    {
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.749919358889262,
      "eval_nll_ablated": 2.7591281652450563,
      "label_use_gap": 0.009208806355794241,
      "perplexity_per_byte_labeled": 15.641370495838729,
      "perplexity_per_byte_ablated": 15.786074099008246
    },
    {
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6191895484924315,
      "eval_nll_ablated": 2.6372561415036517,
      "label_use_gap": 0.018066593011220267,
      "perplexity_per_byte_labeled": 13.72459595700141,
      "perplexity_per_byte_ablated": 13.97480606275897
    },
    {
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5264413515726725,
      "eval_nll_ablated": 2.546364736557007,
      "label_use_gap": 0.019923384984334547,
      "perplexity_per_byte_labeled": 12.508912016257337,
      "perplexity_per_byte_ablated": 12.760631107857774
    }
  ]
}