{
  "experiment": "H1 real-text conditional byte-model pilot v11 (replication of the v10 constant gap)",
  "completed_utc": "2026-11-18T04:07:33.603817+00:00",
  "smoke_test_not_protocol": false,
  "protocol": {
    "dataset": "SetFit/20_newsgroups",
    "revision": "f1b91292074e7cfb69be58b642d583ec262f30ed",
    "dataset_split": "train only; official test unused",
    "selected_data_sha256": "72325b82d18ed20a73b97086b6447e027c820a42c3ce4440cf75e384875095c1",
    "categories": [
      "comp.graphics",
      "comp.os.ms-windows.misc",
      "comp.sys.ibm.pc.hardware",
      "comp.sys.mac.hardware",
      "rec.autos",
      "rec.motorcycles",
      "rec.sport.baseball",
      "rec.sport.hockey"
    ],
    "sizes_train_per_category": [
      30,
      100
    ],
    "subsets_nested": true,
    "validation_per_category": 80,
    "validation_identical_across_sizes": true,
    "validation_messages": 640,
    "bytes_per_example": 96,
    "seed_values": [
      7079,
      8081,
      9091,
      10093,
      11099
    ],
    "steps": [
      100,
      200,
      400,
      800
    ],
    "total_updates": 800,
    "batch_size": 24,
    "conditions": [
      "none",
      "hierarchy",
      "scrambled_1",
      "scrambled_2",
      "scrambled_3",
      "scrambled_4",
      "scrambled_5"
    ],
    "permutation_seeds_selected": [
      42424,
      55555,
      66666,
      77777,
      88888
    ],
    "permutation_seeds_rejected": [],
    "permutation_seeds_primary": [
      42424,
      55555,
      66666,
      77777,
      88888
    ],
    "permutation_seeds_fallback": [
      99991,
      12121,
      23232,
      34343
    ],
    "scrambled_permutations": {
      "scrambled_1": [
        3,
        7,
        4,
        1,
        2,
        6,
        0,
        5
      ],
      "scrambled_2": [
        6,
        0,
        3,
        4,
        2,
        7,
        5,
        1
      ],
      "scrambled_3": [
        4,
        0,
        3,
        1,
        6,
        2,
        7,
        5
      ],
      "scrambled_4": [
        6,
        2,
        0,
        7,
        3,
        4,
        1,
        5
      ],
      "scrambled_5": [
        0,
        6,
        7,
        1,
        3,
        5,
        4,
        2
      ]
    },
    "permutation_rejection_rule": "rejected if categories 0-3 all fall in the same code//4 group (comp/rec split preserved)",
    "same_batch_sequence_per_size_and_seed_across_conditions": true,
    "batch_generator_seed": "seed + 30000, per (size, seed)",
    "evaluations": "each checkpoint: (a) eval_nll_labeled with the condition label prefix; (b) eval_nll_ablated with dummy prefix [256, 256] for all conditions; label_use_gap = ablated - labeled",
    "metric": "validation NLL in nats per byte, lower is better",
    "model": "randomly initialized 2-layer transformer width 64, 4 heads, FF 128, dropout 0.2 in training only (eval mode at every evaluation)",
    "dropout": 0.2,
    "weight_decay": 0.1,
    "learning_rate": 0.001,
    "regularisation_chosen_before_run": true,
    "v9_reference": "v9 used dropout 0 and weight decay 0.01",
    "replication_of": "v10",
    "preregistered_cells": [
      {
        "train_per_category": 30,
        "step": 200
      },
      {
        "train_per_category": 30,
        "step": 400
      },
      {
        "train_per_category": 100,
        "step": 200
      },
      {
        "train_per_category": 100,
        "step": 400
      }
    ],
    "new_seeds_and_permutations_vs_v10": true,
    "v10_seeds_not_reused": [
      2027,
      3037,
      4049,
      5051,
      6067
    ],
    "primary_checkpoint": 800,
    "checkpoint_selection": "none; all checkpoints are reported and the primary criterion is fixed in advance at 800 updates",
    "limitations": "a single regularisation setting (dropout 0.2, weight decay 0.1), unchanged from v10 and not tuned; five new seeds and five new permutations, but still only two groups of four categories, short prefixes, a tiny model; one historical text dataset (20 Newsgroups); no significance test is run or claimed; no data-savings percentage is estimated; labels present at labeled evaluation; the four cells were chosen AFTER seeing v10, so this replication tests a post hoc observation; the cells share seeds across the two checkpoints and are not independent; a null result is informative; not a pretrained-LLM or general data-savings result"
  },
  "environment": {
    "device": "cuda",
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition MIG 2g.48gb",
    "torch": "2.13.0+cu130",
    "cuda": "13.0",
    "python": "3.12.12",
    "code_sha256": "a4f132247c0979327c16a96ee7b260b2228e7017ea602c7a8261002c2a09abcc"
  },
  "summary": {
    "30": {
      "100": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.9652166334788004,
          "hierarchy": 2.9675066471099854,
          "scrambled_1": 2.9672450383504234,
          "scrambled_2": 2.9670430533091228,
          "scrambled_3": 2.966737003326416,
          "scrambled_4": 2.9671204821268717,
          "scrambled_5": 2.9677534071604414,
          "scrambled_mean": 2.967179796854655
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": 0.0003268502553302355,
          "min": -0.0002517573038738341,
          "max": 0.0012115351359045512,
          "per_seed": {
            "7079": 0.00018351236979174956,
            "8081": 0.00045557022094699917,
            "9091": 0.0012115351359045512,
            "10093": 3.539085388171159e-05,
            "11099": -0.0002517573038738341
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 3.0,
          "per_seed": {
            "7079": 3,
            "8081": 3,
            "9091": 5,
            "10093": 3,
            "11099": 1
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 4.0,
          "per_seed": {
            "7079": 4,
            "8081": 4,
            "9091": 6,
            "10093": 4,
            "11099": 2
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0014674441019693952,
          "per_seed": {
            "7079": 0.001591062545776456,
            "8081": 0.001971228917439838,
            "9091": 0.0011840820312500888,
            "10093": 0.0011015733083086232,
            "11099": 0.0014892737070719697
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 1
      },
      "200": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.7753422848383584,
          "hierarchy": 2.776781067848206,
          "scrambled_1": 2.7768728510538736,
          "scrambled_2": 2.7764777692159015,
          "scrambled_3": 2.776819758415222,
          "scrambled_4": 2.7770663483937583,
          "scrambled_5": 2.77696528116862,
          "scrambled_mean": 2.7768404016494754
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -5.9333801269723094e-05,
          "min": -0.0008394400278728398,
          "max": 0.0009073638916015092,
          "per_seed": {
            "7079": -0.0008394400278728398,
            "8081": 0.0009073638916015092,
            "9091": 0.00037696997324632164,
            "10093": -0.0002431758244836324,
            "11099": -0.0004983870188399742
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 2.0,
          "per_seed": {
            "7079": 0,
            "8081": 4,
            "9091": 3,
            "10093": 2,
            "11099": 1
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 3.0,
          "per_seed": {
            "7079": 1,
            "8081": 5,
            "9091": 4,
            "10093": 3,
            "11099": 2
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0014745934804280658,
          "per_seed": {
            "7079": 0.001035205523173044,
            "8081": 0.001863066355387577,
            "9091": 0.0013906319936114464,
            "10093": 0.0021683931350704455,
            "11099": 0.0009156703948978162
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 3
      },
      "400": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.721515676180522,
          "hierarchy": 2.7236231072743733,
          "scrambled_1": 2.724681040445964,
          "scrambled_2": 2.7243461497624715,
          "scrambled_3": 2.7257093874613445,
          "scrambled_4": 2.7248921394348145,
          "scrambled_5": 2.724917254447937,
          "scrambled_mean": 2.724909194310506
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0012860870361326882,
          "min": -0.0020701503753661754,
          "max": -0.0005923366546629616,
          "per_seed": {
            "7079": -0.0005923366546629616,
            "8081": -0.0020701503753661754,
            "9091": -0.0009620904922482687,
            "10093": -0.0016477282842002339,
            "11099": -0.0011581293741858012
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.0,
          "per_seed": {
            "7079": 1,
            "8081": 0,
            "9091": 2,
            "10093": 1,
            "11099": 1
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.0,
          "per_seed": {
            "7079": 2,
            "8081": 1,
            "9091": 3,
            "10093": 2,
            "11099": 2
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0036676565806073036,
          "per_seed": {
            "7079": 0.0015002091725668798,
            "8081": 0.003025197982788441,
            "9091": 0.005573693911234745,
            "10093": 0.004838196436564246,
            "11099": 0.003400985399882206
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 5
      },
      "800": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.8341540304819746,
          "hierarchy": 2.840365804036458,
          "scrambled_1": 2.8415358606974284,
          "scrambled_2": 2.839552090962728,
          "scrambled_3": 2.8429165108998617,
          "scrambled_4": 2.8431118710835777,
          "scrambled_5": 2.842399581273397,
          "scrambled_mean": 2.8419031829833985
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0015373789469401266,
          "min": -0.004953737258911239,
          "max": 0.0011446094512939098,
          "per_seed": {
            "7079": -0.002541793187459618,
            "8081": -0.0016947555541992898,
            "9091": 0.0011446094512939098,
            "10093": -0.004953737258911239,
            "11099": 0.0003587818145756039
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 2.0,
          "per_seed": {
            "7079": 2,
            "8081": 2,
            "9091": 3,
            "10093": 0,
            "11099": 3
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 3.0,
          "per_seed": {
            "7079": 3,
            "8081": 3,
            "9091": 4,
            "10093": 1,
            "11099": 4
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.009906762440999462,
          "per_seed": {
            "7079": 0.013525946935017963,
            "8081": 0.009052578608195194,
            "9091": 0.008196989695231416,
            "10093": 0.007968918482462506,
            "11099": 0.010789378484090228
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 3
      }
    },
    "100": {
      "100": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.9740050792694093,
          "hierarchy": 2.975836992263794,
          "scrambled_1": 2.9764345932006835,
          "scrambled_2": 2.9762773927052817,
          "scrambled_3": 2.9764703496297202,
          "scrambled_4": 2.976891098022461,
          "scrambled_5": 2.9766897360483804,
          "scrambled_mean": 2.9765526339213055
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0007156416575113233,
          "min": -0.0014518324534096294,
          "max": 0.00027834256490111287,
          "per_seed": {
            "7079": 0.00027834256490111287,
            "8081": -0.0002456633249918738,
            "9091": -0.0013348770141603516,
            "10093": -0.0008241780598958748,
            "11099": -0.0014518324534096294
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 0.8,
          "per_seed": {
            "7079": 3,
            "8081": 1,
            "9091": 0,
            "10093": 0,
            "11099": 0
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 1.8,
          "per_seed": {
            "7079": 4,
            "8081": 2,
            "9091": 1,
            "10093": 1,
            "11099": 1
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0011535803476969697,
          "per_seed": {
            "7079": 0.0013712406158448154,
            "8081": 0.0008618831634521484,
            "9091": 0.0014299710591636483,
            "10093": 0.0007158756256102627,
            "11099": 0.0013889312744139737
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 4
      },
      "200": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.761993743578593,
          "hierarchy": 2.763269944190979,
          "scrambled_1": 2.7635736656188965,
          "scrambled_2": 2.764300858179728,
          "scrambled_3": 2.7640031639734906,
          "scrambled_4": 2.7643337106704715,
          "scrambled_5": 2.7643110545476275,
          "scrambled_mean": 2.7641044905980428
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0008345464070637299,
          "min": -0.0014686981836953272,
          "max": -0.00027628898620601916,
          "per_seed": {
            "7079": -0.000636876424153332,
            "8081": -0.00027628898620601916,
            "9091": -0.0005010732014971708,
            "10093": -0.0014686981836953272,
            "11099": -0.0012897952397668
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 0.4,
          "per_seed": {
            "7079": 1,
            "8081": 1,
            "9091": 0,
            "10093": 0,
            "11099": 0
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 1.4,
          "per_seed": {
            "7079": 2,
            "8081": 2,
            "9091": 1,
            "10093": 1,
            "11099": 1
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0013625923792520034,
          "per_seed": {
            "7079": 0.0013607104619342003,
            "8081": 0.002374553680419833,
            "9091": 0.0006726662317912968,
            "10093": 0.0013653755187985617,
            "11099": 0.001039656003316125
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 5
      },
      "400": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.647576964696248,
          "hierarchy": 2.647737455368042,
          "scrambled_1": 2.6482081540425617,
          "scrambled_2": 2.6483685223261513,
          "scrambled_3": 2.648333338101705,
          "scrambled_4": 2.6481064526240035,
          "scrambled_5": 2.6486815579732257,
          "scrambled_mean": 2.64833960501353
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.000602149645487593,
          "min": -0.0009538078308106712,
          "max": -0.00023240884145092622,
          "per_seed": {
            "7079": -0.00023240884145092622,
            "8081": -0.0008490085601806641,
            "9091": -0.0005687681833905778,
            "10093": -0.0004067548116051256,
            "11099": -0.0009538078308106712
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 0.4,
          "per_seed": {
            "7079": 1,
            "8081": 0,
            "9091": 0,
            "10093": 1,
            "11099": 0
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 1.4,
          "per_seed": {
            "7079": 2,
            "8081": 1,
            "9091": 1,
            "10093": 2,
            "11099": 1
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0011680014928181492,
          "per_seed": {
            "7079": 0.0007587512334188418,
            "8081": 0.0018639246622722538,
            "9091": 0.0008705139160154474,
            "10093": 0.0014138936996457296,
            "11099": 0.0009329239527384736
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 5
      },
      "800": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.5921733538309732,
          "hierarchy": 2.5907294066747033,
          "scrambled_1": 2.591126624743144,
          "scrambled_2": 2.590968022346497,
          "scrambled_3": 2.5913943354288738,
          "scrambled_4": 2.5909655984242757,
          "scrambled_5": 2.592439111073812,
          "scrambled_mean": 2.5913787384033204
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0006493317286174083,
          "min": -0.001794985135396221,
          "max": 0.0009465837478632722,
          "per_seed": {
            "7079": 0.0009465837478632722,
            "8081": -0.001794985135396221,
            "9091": -0.0011478074391679804,
            "10093": -0.0004496908187867632,
            "11099": -0.000800758997599349
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.2,
          "per_seed": {
            "7079": 4,
            "8081": 0,
            "9091": 0,
            "10093": 1,
            "11099": 1
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.2,
          "per_seed": {
            "7079": 5,
            "8081": 1,
            "9091": 1,
            "10093": 2,
            "11099": 2
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0022700675328572117,
          "per_seed": {
            "7079": 0.0036103328069052587,
            "8081": 0.004296294848124216,
            "9091": 0.0008780241012571466,
            "10093": 0.0009408473968504083,
            "11099": 0.0016248385111490293
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 4
      }
    }
  },
  "replication_cells": {
    "cells": {
      "30_200": {
        "train_per_category": 30,
        "step": 200,
        "mean_paired_diff": -5.9333801269723094e-05,
        "seeds_hierarchy_better": 3,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.0014745934804280658,
        "sign_negative": true,
        "wins_at_least_4_of_5": false,
        "material": false
      },
      "30_400": {
        "train_per_category": 30,
        "step": 400,
        "mean_paired_diff": -0.0012860870361326882,
        "seeds_hierarchy_better": 5,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.0036676565806073036,
        "sign_negative": true,
        "wins_at_least_4_of_5": true,
        "material": false
      },
      "100_200": {
        "train_per_category": 100,
        "step": 200,
        "mean_paired_diff": -0.0008345464070637299,
        "seeds_hierarchy_better": 5,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.0013625923792520034,
        "sign_negative": true,
        "wins_at_least_4_of_5": true,
        "material": false
      },
      "100_400": {
        "train_per_category": 100,
        "step": 400,
        "mean_paired_diff": -0.000602149645487593,
        "seeds_hierarchy_better": 5,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.0011680014928181492,
        "sign_negative": true,
        "wins_at_least_4_of_5": true,
        "material": false
      }
    },
    "missing_cells": [],
    "n_cells_sign_negative": 4,
    "n_cells_wins_at_least_4_of_5": 3,
    "n_cells_material": 0
  },
  "summary_label_use": {
    "30": {
      "100": {
        "none": 0.0,
        "hierarchy": 0.009147793451945052,
        "scrambled_1": 0.009500118891398124,
        "scrambled_2": 0.009711068471272722,
        "scrambled_3": 0.00920822461446118,
        "scrambled_4": 0.009318552017211879,
        "scrambled_5": 0.008893785476684535,
        "scrambled_mean": 0.009326349894205688
      },
      "200": {
        "none": 0.0,
        "hierarchy": 0.008630337715148872,
        "scrambled_1": 0.007672653198242152,
        "scrambled_2": 0.00829879283905024,
        "scrambled_3": 0.007573920885721996,
        "scrambled_4": 0.008317529360453246,
        "scrambled_5": 0.007883346875508578,
        "scrambled_mean": 0.007949248631795242
      },
      "400": {
        "none": 0.0,
        "hierarchy": 0.01457290331522625,
        "scrambled_1": 0.011372235616048165,
        "scrambled_2": 0.015328660011291451,
        "scrambled_3": 0.014444616635640362,
        "scrambled_4": 0.013685970306396645,
        "scrambled_5": 0.012312509218851719,
        "scrambled_mean": 0.013428798357645669
      },
      "800": {
        "none": 0.0,
        "hierarchy": 0.032198235193888355,
        "scrambled_1": 0.030165863037109376,
        "scrambled_2": 0.02900349934895834,
        "scrambled_3": 0.0328964678446451,
        "scrambled_4": 0.025991417566935127,
        "scrambled_5": 0.02442124366760252,
        "scrambled_mean": 0.028495698293050094
      }
    },
    "100": {
      "100": {
        "none": 0.0,
        "hierarchy": 0.009537019729614293,
        "scrambled_1": 0.009278602600097852,
        "scrambled_2": 0.008767162958780972,
        "scrambled_3": 0.00848538080851231,
        "scrambled_4": 0.009226430257161588,
        "scrambled_5": 0.009086491266886477,
        "scrambled_mean": 0.00896881357828784
      },
      "200": {
        "none": 0.0,
        "hierarchy": 0.008021157582601024,
        "scrambled_1": 0.007657945950825962,
        "scrambled_2": 0.007025392850240131,
        "scrambled_3": 0.007245206832885831,
        "scrambled_4": 0.007421795527140329,
        "scrambled_5": 0.00733801523844404,
        "scrambled_mean": 0.007337671279907258
      },
      "400": {
        "none": 0.0,
        "hierarchy": 0.009718680381774902,
        "scrambled_1": 0.008884539604186958,
        "scrambled_2": 0.009266835848490373,
        "scrambled_3": 0.009162979125976457,
        "scrambled_4": 0.008993298212687151,
        "scrambled_5": 0.008759365081787074,
        "scrambled_mean": 0.009013403574625602
      },
      "800": {
        "none": 0.0,
        "hierarchy": 0.016476054191589196,
        "scrambled_1": 0.012413007418314503,
        "scrambled_2": 0.015076317787170446,
        "scrambled_3": 0.014405822753906339,
        "scrambled_4": 0.014903483390808159,
        "scrambled_5": 0.013334150314330984,
        "scrambled_mean": 0.014026556332906085
      }
    }
  },
  "elapsed_seconds": 226.4974012829989,
  "runs": [
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.946975008646647,
      "eval_nll_ablated": 2.946975008646647,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.048245710538566,
      "perplexity_per_byte_ablated": 19.048245710538566
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.766776665051778,
      "eval_nll_ablated": 2.766776665051778,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.9072768022716,
      "perplexity_per_byte_ablated": 15.9072768022716
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7228023846944174,
      "eval_nll_ablated": 2.7228023846944174,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.222923011657842,
      "perplexity_per_byte_ablated": 15.222923011657842
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8371204376220702,
      "eval_nll_ablated": 2.8371204376220702,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 17.0665505155051,
      "perplexity_per_byte_ablated": 17.0665505155051
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9484631220499673,
      "eval_nll_ablated": 2.960525862375895,
      "label_use_gap": 0.012062740325927912,
      "perplexity_per_byte_labeled": 19.076612761749857,
      "perplexity_per_byte_ablated": 19.308122501493852
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.767889984448751,
      "eval_nll_ablated": 2.776637323697408,
      "label_use_gap": 0.008747339248657227,
      "perplexity_per_byte_labeled": 15.924996544124465,
      "perplexity_per_byte_ablated": 16.064908929854937
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7279342810312905,
      "eval_nll_ablated": 2.743438959121704,
      "label_use_gap": 0.01550467809041356,
      "perplexity_per_byte_labeled": 15.301246276038926,
      "perplexity_per_byte_ablated": 15.54033588802139
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8450266202290853,
      "eval_nll_ablated": 2.8615381876627604,
      "label_use_gap": 0.0165115674336751,
      "perplexity_per_byte_labeled": 17.20201658445271,
      "perplexity_per_byte_ablated": 17.488406709654612
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.94760955174764,
      "eval_nll_ablated": 2.9591544310251874,
      "label_use_gap": 0.011544879277547437,
      "perplexity_per_byte_labeled": 19.0603364790914,
      "perplexity_per_byte_ablated": 19.281660886263825
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.769102907180786,
      "eval_nll_ablated": 2.7775373617808023,
      "label_use_gap": 0.008434454600016128,
      "perplexity_per_byte_labeled": 15.94432405345861,
      "perplexity_per_byte_ablated": 16.079374468489394
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7279410044352215,
      "eval_nll_ablated": 2.738139828046163,
      "label_use_gap": 0.010198823610941599,
      "perplexity_per_byte_labeled": 15.301349152844127,
      "perplexity_per_byte_ablated": 15.458203418748244
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8506505330403646,
      "eval_nll_ablated": 2.866861073176066,
      "label_use_gap": 0.016210540135701557,
      "perplexity_per_byte_labeled": 17.299031772679072,
      "perplexity_per_byte_ablated": 17.58174368702545
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9488135019938153,
      "eval_nll_ablated": 2.9593231201171877,
      "label_use_gap": 0.010509618123372366,
      "perplexity_per_byte_labeled": 19.083297995375627,
      "perplexity_per_byte_ablated": 19.284913766485978
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.768067701657613,
      "eval_nll_ablated": 2.775900300343831,
      "label_use_gap": 0.007832598686217995,
      "perplexity_per_byte_labeled": 15.927826941559147,
      "perplexity_per_byte_ablated": 16.05307307898557
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7293832937876386,
      "eval_nll_ablated": 2.7430915594100953,
      "label_use_gap": 0.01370826562245675,
      "perplexity_per_byte_labeled": 15.323434048381317,
      "perplexity_per_byte_ablated": 15.534938117461842
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8518270174662272,
      "eval_nll_ablated": 2.8664915402730307,
      "label_use_gap": 0.014664522806803415,
      "perplexity_per_byte_labeled": 17.31939579076835,
      "perplexity_per_byte_ablated": 17.575247854526197
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9479933738708497,
      "eval_nll_ablated": 2.959713347752889,
      "label_use_gap": 0.01171997388203927,
      "perplexity_per_byte_labeled": 19.06765366206642,
      "perplexity_per_byte_ablated": 19.292440741311054
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.768535868326823,
      "eval_nll_ablated": 2.7764453570048016,
      "label_use_gap": 0.007909488677978693,
      "perplexity_per_byte_labeled": 15.935285565049352,
      "perplexity_per_byte_ablated": 16.06182529840738
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.728227241834005,
      "eval_nll_ablated": 2.7434767802556355,
      "label_use_gap": 0.015249538421630682,
      "perplexity_per_byte_labeled": 15.305729598117233,
      "perplexity_per_byte_ablated": 15.540923652261235
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8443140665690105,
      "eval_nll_ablated": 2.866296577453613,
      "label_use_gap": 0.02198251088460257,
      "perplexity_per_byte_labeled": 17.189763590551056,
      "perplexity_per_byte_ablated": 17.5718216686528
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9477810064951577,
      "eval_nll_ablated": 2.959357722600301,
      "label_use_gap": 0.011576716105143436,
      "perplexity_per_byte_labeled": 19.063604744441825,
      "perplexity_per_byte_ablated": 19.285581083934282
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.768943937619527,
      "eval_nll_ablated": 2.7765085061391193,
      "label_use_gap": 0.007564568519592196,
      "perplexity_per_byte_labeled": 15.94178959271565,
      "perplexity_per_byte_ablated": 16.062839620796993
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.729198463757833,
      "eval_nll_ablated": 2.7417519807815554,
      "label_use_gap": 0.012553517023722538,
      "perplexity_per_byte_labeled": 15.320602079333952,
      "perplexity_per_byte_ablated": 15.514141778639386
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8522881984710695,
      "eval_nll_ablated": 2.869075314203898,
      "label_use_gap": 0.01678711573282854,
      "perplexity_per_byte_labeled": 17.327385009218695,
      "perplexity_per_byte_ablated": 17.620717037510488
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9492006142934164,
      "eval_nll_ablated": 2.9595738887786864,
      "label_use_gap": 0.010373274485270034,
      "perplexity_per_byte_labeled": 19.090686804803823,
      "perplexity_per_byte_ablated": 19.289750424914153
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7689967075983684,
      "eval_nll_ablated": 2.77514119942983,
      "label_use_gap": 0.006144491831461707,
      "perplexity_per_byte_labeled": 15.942630862811859,
      "perplexity_per_byte_ablated": 16.04089180053353
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7278830846150717,
      "eval_nll_ablated": 2.7390017032623293,
      "label_use_gap": 0.011118618647257605,
      "perplexity_per_byte_labeled": 15.30046292711841,
      "perplexity_per_byte_ablated": 15.471532204211162
    },
    {
      "train_per_category": 30,
      "seed": 7079,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8387622515360516,
      "eval_nll_ablated": 2.85119423866272,
      "label_use_gap": 0.012431987126668265,
      "perplexity_per_byte_labeled": 17.094593630093673,
      "perplexity_per_byte_ablated": 17.30843991091224
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9752046267191568,
      "eval_nll_ablated": 2.9752046267191568,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.593632220088324,
      "perplexity_per_byte_ablated": 19.593632220088324
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7900450627009072,
      "eval_nll_ablated": 2.7900450627009072,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.28175348504497,
      "perplexity_per_byte_ablated": 16.28175348504497
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.725874670346578,
      "eval_nll_ablated": 2.725874670346578,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.269764097364105,
      "perplexity_per_byte_ablated": 15.269764097364105
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8422847588857016,
      "eval_nll_ablated": 2.8422847588857016,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 17.154915641786488,
      "perplexity_per_byte_ablated": 17.154915641786488
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9697853565216064,
      "eval_nll_ablated": 2.9838306585947674,
      "label_use_gap": 0.014045302073161015,
      "perplexity_per_byte_labeled": 19.487736231590464,
      "perplexity_per_byte_ablated": 19.763378582398495
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7892448902130127,
      "eval_nll_ablated": 2.807745361328125,
      "label_use_gap": 0.018500471115112216,
      "perplexity_per_byte_labeled": 16.268730484869643,
      "perplexity_per_byte_ablated": 16.572511040515597
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7236042896906536,
      "eval_nll_ablated": 2.7486319541931152,
      "label_use_gap": 0.025027664502461633,
      "perplexity_per_byte_labeled": 15.235135245549472,
      "perplexity_per_byte_ablated": 15.621246678536115
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8374219576517743,
      "eval_nll_ablated": 2.9054582595825194,
      "label_use_gap": 0.06803630193074506,
      "perplexity_per_byte_labeled": 17.071697198198446,
      "perplexity_per_byte_ablated": 18.273615793300284
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.96975417137146,
      "eval_nll_ablated": 2.9864333470662436,
      "label_use_gap": 0.0166791756947835,
      "perplexity_per_byte_labeled": 19.48712851308602,
      "perplexity_per_byte_ablated": 19.81488349654099
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7880396048227944,
      "eval_nll_ablated": 2.804456528027852,
      "label_use_gap": 0.01641692320505772,
      "perplexity_per_byte_labeled": 16.249133833849775,
      "perplexity_per_byte_ablated": 16.518096343817373
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7267983754475913,
      "eval_nll_ablated": 2.750910425186157,
      "label_use_gap": 0.024112049738565844,
      "perplexity_per_byte_labeled": 15.283875372677256,
      "perplexity_per_byte_ablated": 15.656879815086747
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8422853628794353,
      "eval_nll_ablated": 2.9189152081807452,
      "label_use_gap": 0.0766298453013099,
      "perplexity_per_byte_labeled": 17.154926003251166,
      "perplexity_per_byte_ablated": 18.52118492829426
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9682700634002686,
      "eval_nll_ablated": 2.987939484914144,
      "label_use_gap": 0.019669421513875296,
      "perplexity_per_byte_labeled": 19.45822896065663,
      "perplexity_per_byte_ablated": 19.84474992836266
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7874186197916666,
      "eval_nll_ablated": 2.807685073216756,
      "label_use_gap": 0.02026645342508937,
      "perplexity_per_byte_labeled": 16.23904649733681,
      "perplexity_per_byte_ablated": 16.57151194524141
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.723773177464803,
      "eval_nll_ablated": 2.752853600184123,
      "label_use_gap": 0.029080422719319987,
      "perplexity_per_byte_labeled": 15.237708490918681,
      "perplexity_per_byte_ablated": 15.687333451279237
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8332491079966227,
      "eval_nll_ablated": 2.8956230163574217,
      "label_use_gap": 0.062373908360799035,
      "perplexity_per_byte_labeled": 17.000607997859046,
      "perplexity_per_byte_ablated": 18.09477126873401
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.968519989649455,
      "eval_nll_ablated": 2.985169998804728,
      "label_use_gap": 0.01665000915527326,
      "perplexity_per_byte_labeled": 19.463092690598153,
      "perplexity_per_byte_ablated": 19.78986620402741
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7887900352478026,
      "eval_nll_ablated": 2.80590136051178,
      "label_use_gap": 0.017111325263977406,
      "perplexity_per_byte_labeled": 16.261332254719346,
      "perplexity_per_byte_ablated": 16.541979475394978
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.72585391998291,
      "eval_nll_ablated": 2.755096419652303,
      "label_use_gap": 0.029242499669392963,
      "perplexity_per_byte_labeled": 15.269447247493348,
      "perplexity_per_byte_ablated": 15.722556793180926
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.842301686604818,
      "eval_nll_ablated": 2.912411181131999,
      "label_use_gap": 0.07010949452718096,
      "perplexity_per_byte_labeled": 17.1552060378378,
      "perplexity_per_byte_ablated": 18.40111353760725
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9698634147644043,
      "eval_nll_ablated": 2.9861350536346434,
      "label_use_gap": 0.01627163887023908,
      "perplexity_per_byte_labeled": 19.489257469408617,
      "perplexity_per_byte_ablated": 19.808973728412376
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.789281686147054,
      "eval_nll_ablated": 2.809803501764933,
      "label_use_gap": 0.020521815617879025,
      "perplexity_per_byte_labeled": 16.269329119017087,
      "perplexity_per_byte_ablated": 16.6066547198185
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7263521353403726,
      "eval_nll_ablated": 2.7544395128885903,
      "label_use_gap": 0.028087377548217773,
      "perplexity_per_byte_labeled": 15.27705661600674,
      "perplexity_per_byte_ablated": 15.712231930887851
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.841474151611328,
      "eval_nll_ablated": 2.903669500350952,
      "label_use_gap": 0.062195348739623846,
      "perplexity_per_byte_labeled": 17.141015376965022,
      "perplexity_per_byte_ablated": 18.240957911607428
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9702412923177084,
      "eval_nll_ablated": 2.984855286280314,
      "label_use_gap": 0.014613993962605765,
      "perplexity_per_byte_labeled": 19.49662341396177,
      "perplexity_per_byte_ablated": 19.783639065207208
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7881576855977377,
      "eval_nll_ablated": 2.808074386914571,
      "label_use_gap": 0.01991670131683332,
      "perplexity_per_byte_labeled": 16.25105265745089,
      "perplexity_per_byte_ablated": 16.577964717830074
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7255945920944216,
      "eval_nll_ablated": 2.749983874956767,
      "label_use_gap": 0.02438928286234532,
      "perplexity_per_byte_labeled": 15.265487967378334,
      "perplexity_per_byte_ablated": 15.642379648106424
    },
    {
      "train_per_category": 30,
      "seed": 8081,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.836273256937663,
      "eval_nll_ablated": 2.8865490277608234,
      "label_use_gap": 0.05027577082316048,
      "perplexity_per_byte_labeled": 17.05209818629037,
      "perplexity_per_byte_ablated": 17.931322209527373
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9481530666351317,
      "eval_nll_ablated": 2.9481530666351317,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.070698871530574,
      "perplexity_per_byte_ablated": 19.070698871530574
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.767089509963989,
      "eval_nll_ablated": 2.767089509963989,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.912254091405519,
      "perplexity_per_byte_ablated": 15.912254091405519
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7235944986343386,
      "eval_nll_ablated": 2.7235944986343386,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.23498607821257,
      "perplexity_per_byte_ablated": 15.23498607821257
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8235311031341555,
      "eval_nll_ablated": 2.8235311031341555,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.836196178017047,
      "perplexity_per_byte_ablated": 16.836196178017047
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9512529691060383,
      "eval_nll_ablated": 2.955063470204671,
      "label_use_gap": 0.0038105010986329013,
      "perplexity_per_byte_labeled": 19.12990790178036,
      "perplexity_per_byte_ablated": 19.202941495781797
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7703888177871705,
      "eval_nll_ablated": 2.7741463661193846,
      "label_use_gap": 0.003757548332214089,
      "perplexity_per_byte_labeled": 15.964840217019674,
      "perplexity_per_byte_ablated": 16.024941722191198
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.727698795000712,
      "eval_nll_ablated": 2.738321137428284,
      "label_use_gap": 0.010622342427571763,
      "perplexity_per_byte_labeled": 15.297643470512313,
      "perplexity_per_byte_ablated": 15.461006390153624
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.844326941172282,
      "eval_nll_ablated": 2.864259783426921,
      "label_use_gap": 0.01993284225463876,
      "perplexity_per_byte_labeled": 17.18998490336227,
      "perplexity_per_byte_ablated": 17.53606791112033
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9507097721099855,
      "eval_nll_ablated": 2.9549057324727377,
      "label_use_gap": 0.00419596036275216,
      "perplexity_per_byte_labeled": 19.119519415026183,
      "perplexity_per_byte_ablated": 19.199912706227277
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7701669692993165,
      "eval_nll_ablated": 2.7725284973780315,
      "label_use_gap": 0.002361528078715036,
      "perplexity_per_byte_labeled": 15.961298834198434,
      "perplexity_per_byte_ablated": 15.999036431227694
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7271808942159015,
      "eval_nll_ablated": 2.7350377162297566,
      "label_use_gap": 0.00785682201385507,
      "perplexity_per_byte_labeled": 15.289722860175376,
      "perplexity_per_byte_ablated": 15.410324644313665
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8396018028259276,
      "eval_nll_ablated": 2.8594971974690755,
      "label_use_gap": 0.019895394643147934,
      "perplexity_per_byte_labeled": 17.108951444443168,
      "perplexity_per_byte_ablated": 17.45274944350026
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.950091552734375,
      "eval_nll_ablated": 2.954517952601115,
      "label_use_gap": 0.004426399866739761,
      "perplexity_per_byte_labeled": 19.107703010612884,
      "perplexity_per_byte_ablated": 19.192468809932745
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7705840428670245,
      "eval_nll_ablated": 2.7735221068064373,
      "label_use_gap": 0.0029380639394127606,
      "perplexity_per_byte_labeled": 15.967957258478332,
      "perplexity_per_byte_ablated": 16.014941124889454
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.728212078412374,
      "eval_nll_ablated": 2.7394872188568113,
      "label_use_gap": 0.011275140444437337,
      "perplexity_per_byte_labeled": 15.30549751264558,
      "perplexity_per_byte_ablated": 15.479045698178489
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8412954171498614,
      "eval_nll_ablated": 2.862965393066406,
      "label_use_gap": 0.021669975916544626,
      "perplexity_per_byte_labeled": 17.13795196058982,
      "perplexity_per_byte_ablated": 17.513384077889512
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9495256900787354,
      "eval_nll_ablated": 2.954185152053833,
      "label_use_gap": 0.004659461975097479,
      "perplexity_per_byte_labeled": 19.096893733615644,
      "perplexity_per_byte_ablated": 19.18608260853371
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7695343494415283,
      "eval_nll_ablated": 2.7728947162628175,
      "label_use_gap": 0.0033603668212891513,
      "perplexity_per_byte_labeled": 15.951204592845242,
      "perplexity_per_byte_ablated": 16.004896653503724
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.732303484280904,
      "eval_nll_ablated": 2.741594131787618,
      "label_use_gap": 0.0092906475067136,
      "perplexity_per_byte_labeled": 15.368246793845124,
      "perplexity_per_byte_ablated": 15.511693080235197
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8468262990315756,
      "eval_nll_ablated": 2.8729364395141603,
      "label_use_gap": 0.026110140482584754,
      "perplexity_per_byte_labeled": 17.23300256310108,
      "perplexity_per_byte_ablated": 17.688884350629856
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9500595569610595,
      "eval_nll_ablated": 2.9549251874287923,
      "label_use_gap": 0.004865630467732807,
      "perplexity_per_byte_labeled": 19.10709165465923,
      "perplexity_per_byte_ablated": 19.200286243318793
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.770580466588338,
      "eval_nll_ablated": 2.773419205347697,
      "label_use_gap": 0.0028387387593591384,
      "perplexity_per_byte_labeled": 15.967900152715234,
      "perplexity_per_byte_ablated": 16.013293248871943
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7288781801859536,
      "eval_nll_ablated": 2.738604521751404,
      "label_use_gap": 0.00972634156545027,
      "perplexity_per_byte_labeled": 15.315695927898437,
      "perplexity_per_byte_ablated": 15.465388417853903
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.840389347076416,
      "eval_nll_ablated": 2.858473602930705,
      "label_use_gap": 0.018084255854288855,
      "perplexity_per_byte_labeled": 17.122430807886026,
      "perplexity_per_byte_ablated": 17.43489404439384
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.949820597966512,
      "eval_nll_ablated": 2.954177157084147,
      "label_use_gap": 0.004356559117634884,
      "perplexity_per_byte_labeled": 19.10252638872611,
      "perplexity_per_byte_ablated": 19.185929216998044
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.769193410873413,
      "eval_nll_ablated": 2.772286558151245,
      "label_use_gap": 0.0030931472778319424,
      "perplexity_per_byte_labeled": 15.945767138963186,
      "perplexity_per_byte_ablated": 15.995166104934947
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7267297903696694,
      "eval_nll_ablated": 2.736857263247172,
      "label_use_gap": 0.010127472877502708,
      "perplexity_per_byte_labeled": 15.28282716284006,
      "perplexity_per_byte_ablated": 15.438389979913362
    },
    {
      "train_per_category": 30,
      "seed": 9091,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.847798792521159,
      "eval_nll_ablated": 2.8682891845703127,
      "label_use_gap": 0.020490392049153705,
      "perplexity_per_byte_labeled": 17.249769697542764,
      "perplexity_per_byte_ablated": 17.606870313055595
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9890621026357014,
      "eval_nll_ablated": 2.9890621026357014,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.86704050586977,
      "perplexity_per_byte_ablated": 19.86704050586977
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7838220675786336,
      "eval_nll_ablated": 2.7838220675786336,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.18074682047827,
      "perplexity_per_byte_ablated": 16.18074682047827
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7153667847315472,
      "eval_nll_ablated": 2.7153667847315472,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.110151228389425,
      "perplexity_per_byte_ablated": 15.110151228389425
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.831649398803711,
      "eval_nll_ablated": 2.831649398803711,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.97343371023372,
      "perplexity_per_byte_ablated": 16.97343371023372
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.999082851409912,
      "eval_nll_ablated": 3.003446038564046,
      "label_use_gap": 0.004363187154134085,
      "perplexity_per_byte_labeled": 20.067123946325978,
      "perplexity_per_byte_ablated": 20.154871854814925
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7876723210016885,
      "eval_nll_ablated": 2.7929465055465696,
      "label_use_gap": 0.005274184544881155,
      "perplexity_per_byte_labeled": 16.243166885734446,
      "perplexity_per_byte_ablated": 16.329062661455865
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7164745648701984,
      "eval_nll_ablated": 2.7245187997817992,
      "label_use_gap": 0.008044234911600778,
      "perplexity_per_byte_labeled": 15.126899228650545,
      "perplexity_per_byte_ablated": 15.24907430320447
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.82873420715332,
      "eval_nll_ablated": 2.8577309449513755,
      "label_use_gap": 0.02899673779805534,
      "perplexity_per_byte_labeled": 16.92402495099542,
      "perplexity_per_byte_ablated": 17.42195068805134
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9988577524820963,
      "eval_nll_ablated": 3.003150192896525,
      "label_use_gap": 0.004292440414428889,
      "perplexity_per_byte_labeled": 20.062607366599014,
      "perplexity_per_byte_ablated": 20.14891000523441
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7879565795262655,
      "eval_nll_ablated": 2.7925469001134235,
      "label_use_gap": 0.0045903205871580255,
      "perplexity_per_byte_labeled": 16.247784800697595,
      "perplexity_per_byte_ablated": 16.322538782873643
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7177426338195803,
      "eval_nll_ablated": 2.723158264160156,
      "label_use_gap": 0.005415630340575905,
      "perplexity_per_byte_labeled": 15.146093347023575,
      "perplexity_per_byte_ablated": 15.228341501473658
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.829314629236857,
      "eval_nll_ablated": 2.85406813621521,
      "label_use_gap": 0.0247535069783531,
      "perplexity_per_byte_labeled": 16.933850880136582,
      "perplexity_per_byte_ablated": 17.358254140219646
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9989124298095704,
      "eval_nll_ablated": 3.003113094965617,
      "label_use_gap": 0.004200665156046668,
      "perplexity_per_byte_labeled": 20.06370436634222,
      "perplexity_per_byte_ablated": 20.148162536228025
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7869094212849936,
      "eval_nll_ablated": 2.791738271713257,
      "label_use_gap": 0.004828850428263198,
      "perplexity_per_byte_labeled": 16.2307797040086,
      "perplexity_per_byte_ablated": 16.309345249502794
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.716825803120931,
      "eval_nll_ablated": 2.7254143873850505,
      "label_use_gap": 0.008588584264119348,
      "perplexity_per_byte_labeled": 15.132213307473869,
      "perplexity_per_byte_ablated": 15.262737302404599
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8302982330322264,
      "eval_nll_ablated": 2.8548298358917235,
      "label_use_gap": 0.02453160285949707,
      "perplexity_per_byte_labeled": 16.95051527436968,
      "perplexity_per_byte_ablated": 17.371480953573315
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.999276701609294,
      "eval_nll_ablated": 3.002622683842977,
      "label_use_gap": 0.003345982233683209,
      "perplexity_per_byte_labeled": 20.071014339368546,
      "perplexity_per_byte_ablated": 20.138284075670946
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7886311133702595,
      "eval_nll_ablated": 2.7920228640238443,
      "label_use_gap": 0.0033917506535847686,
      "perplexity_per_byte_labeled": 16.258748178604627,
      "perplexity_per_byte_ablated": 16.31398742428381
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.720688565572103,
      "eval_nll_ablated": 2.7260605414708454,
      "label_use_gap": 0.005371975898742498,
      "perplexity_per_byte_labeled": 15.190778491717838,
      "perplexity_per_byte_ablated": 15.272602569371
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.837022384007772,
      "eval_nll_ablated": 2.8633920351664224,
      "label_use_gap": 0.02636965115865042,
      "perplexity_per_byte_labeled": 17.06487716058404,
      "perplexity_per_byte_ablated": 17.52085761900117
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.998544422785441,
      "eval_nll_ablated": 3.0025967439015706,
      "label_use_gap": 0.004052321116129409,
      "perplexity_per_byte_labeled": 20.05632214064412,
      "perplexity_per_byte_ablated": 20.137761696537268
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7870025555292766,
      "eval_nll_ablated": 2.7917207717895507,
      "label_use_gap": 0.004718216260274133,
      "perplexity_per_byte_labeled": 16.23229141580543,
      "perplexity_per_byte_ablated": 16.309059839702563
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7158503691355387,
      "eval_nll_ablated": 2.725259526570638,
      "label_use_gap": 0.009409157435099402,
      "perplexity_per_byte_labeled": 15.11746002893397,
      "perplexity_per_byte_ablated": 15.260373885480844
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.834520928064982,
      "eval_nll_ablated": 2.852521228790283,
      "label_use_gap": 0.0180003007253009,
      "perplexity_per_byte_labeled": 17.022243467581,
      "perplexity_per_byte_ablated": 17.33142328578069
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.99964599609375,
      "eval_nll_ablated": 3.0037843386332193,
      "label_use_gap": 0.004138342539469431,
      "perplexity_per_byte_labeled": 20.07842782305658,
      "perplexity_per_byte_ablated": 20.161691402819326
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.789077814420064,
      "eval_nll_ablated": 2.79255047639211,
      "label_use_gap": 0.0034726619720459873,
      "perplexity_per_byte_labeled": 16.26601260087625,
      "perplexity_per_byte_ablated": 16.322597156925582
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7195040941238404,
      "eval_nll_ablated": 2.723878502845764,
      "label_use_gap": 0.004374408721923739,
      "perplexity_per_byte_labeled": 15.17279610023456,
      "perplexity_per_byte_ablated": 15.239313492892704
    },
    {
      "train_per_category": 30,
      "seed": 10093,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8372835477193195,
      "eval_nll_ablated": 2.8606528917948406,
      "label_use_gap": 0.02336934407552116,
      "perplexity_per_byte_labeled": 17.069334469258703,
      "perplexity_per_byte_ablated": 17.472931146694773
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.966688362757365,
      "eval_nll_ablated": 2.966688362757365,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.42747619464606,
      "perplexity_per_byte_ablated": 19.42747619464606
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7689781188964844,
      "eval_nll_ablated": 2.7689781188964844,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.942334512753893,
      "perplexity_per_byte_ablated": 15.942334512753893
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7199400424957276,
      "eval_nll_ablated": 2.7199400424957276,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.17941209800338,
      "perplexity_per_byte_ablated": 15.17941209800338
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8361844539642336,
      "eval_nll_ablated": 2.8361844539642336,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 17.050583976502377,
      "perplexity_per_byte_ablated": 17.050583976502377
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9689489364624024,
      "eval_nll_ablated": 2.980406173070272,
      "label_use_gap": 0.011457236607869348,
      "perplexity_per_byte_labeled": 19.471443112993384,
      "perplexity_per_byte_ablated": 19.695814929930012
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.768709325790405,
      "eval_nll_ablated": 2.775581471125285,
      "label_use_gap": 0.006872145334879676,
      "perplexity_per_byte_labeled": 15.938049899005172,
      "perplexity_per_byte_ablated": 16.047955706067885
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.722403605779012,
      "eval_nll_ablated": 2.7360692024230957,
      "label_use_gap": 0.01366559664408351,
      "perplexity_per_byte_labeled": 15.216853641178878,
      "perplexity_per_byte_ablated": 15.426228382252118
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.84631929397583,
      "eval_nll_ablated": 2.8738330205281577,
      "label_use_gap": 0.027513726552327533,
      "perplexity_per_byte_labeled": 17.22426755820885,
      "perplexity_per_byte_ablated": 17.704750980294005
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9692939440409343,
      "eval_nll_ablated": 2.980082082748413,
      "label_use_gap": 0.010788138707478634,
      "perplexity_per_byte_labeled": 19.47816206741077,
      "perplexity_per_byte_ablated": 19.689432741188764
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7690981944402058,
      "eval_nll_ablated": 2.7756582339604696,
      "label_use_gap": 0.0065600395202638495,
      "perplexity_per_byte_labeled": 15.944248912172675,
      "perplexity_per_byte_ablated": 16.049187639929563
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7237422943115233,
      "eval_nll_ablated": 2.7330201466878257,
      "label_use_gap": 0.009277852376302409,
      "perplexity_per_byte_labeled": 15.23723790969828,
      "perplexity_per_byte_ablated": 15.379264586129766
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8458269755045573,
      "eval_nll_ablated": 2.8591670036315917,
      "label_use_gap": 0.013340028127034387,
      "perplexity_per_byte_labeled": 17.215789820180547,
      "perplexity_per_byte_ablated": 17.446987604501082
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.969127718607585,
      "eval_nll_ablated": 2.9788769563039144,
      "label_use_gap": 0.009749237696329516,
      "perplexity_per_byte_labeled": 19.474924570564877,
      "perplexity_per_byte_ablated": 19.665718777147806
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7694090604782104,
      "eval_nll_ablated": 2.7750370581944783,
      "label_use_gap": 0.005627997716267874,
      "perplexity_per_byte_labeled": 15.94920620814852,
      "perplexity_per_byte_ablated": 16.039221369227178
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7235363960266112,
      "eval_nll_ablated": 2.737527283032735,
      "label_use_gap": 0.013990887006123831,
      "perplexity_per_byte_labeled": 15.234100911508232,
      "perplexity_per_byte_ablated": 15.448737472782511
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.841090679168701,
      "eval_nll_ablated": 2.8628681659698487,
      "label_use_gap": 0.02177748680114755,
      "perplexity_per_byte_labeled": 17.134443530070932,
      "perplexity_per_byte_ablated": 17.51168138518
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9683692614237467,
      "eval_nll_ablated": 2.9780349572499594,
      "label_use_gap": 0.009665695826212684,
      "perplexity_per_byte_labeled": 19.46015927424998,
      "perplexity_per_byte_ablated": 19.649167229713612
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.768607425689697,
      "eval_nll_ablated": 2.774704098701477,
      "label_use_gap": 0.006096673011779963,
      "perplexity_per_byte_labeled": 15.936425892859972,
      "perplexity_per_byte_ablated": 16.033881847183547
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7214737256368,
      "eval_nll_ablated": 2.734542147318522,
      "label_use_gap": 0.013068421681722064,
      "perplexity_per_byte_labeled": 15.202710367944572,
      "perplexity_per_byte_ablated": 15.402689658494973
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.844118118286133,
      "eval_nll_ablated": 2.8640286604563396,
      "label_use_gap": 0.019910542170206824,
      "perplexity_per_byte_labeled": 17.18639561587762,
      "perplexity_per_byte_ablated": 17.532015391345645
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.969354009628296,
      "eval_nll_ablated": 2.9791804631551106,
      "label_use_gap": 0.00982645352681466,
      "perplexity_per_byte_labeled": 19.479332069794165,
      "perplexity_per_byte_ablated": 19.671688363389706
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.769523096084595,
      "eval_nll_ablated": 2.7754674037297566,
      "label_use_gap": 0.005944307645161739,
      "perplexity_per_byte_labeled": 15.951025089256452,
      "perplexity_per_byte_ablated": 16.046125261955922
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.724181548754374,
      "eval_nll_ablated": 2.7328350067138674,
      "label_use_gap": 0.008653457959493238,
      "perplexity_per_byte_labeled": 15.243932404332508,
      "perplexity_per_byte_ablated": 15.376417533044592
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.846886730194092,
      "eval_nll_ablated": 2.8617767969767254,
      "label_use_gap": 0.014890066782633493,
      "perplexity_per_byte_labeled": 17.234044004947073,
      "perplexity_per_byte_ablated": 17.492580104267557
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9698585351308187,
      "eval_nll_ablated": 2.980845292409261,
      "label_use_gap": 0.01098675727844256,
      "perplexity_per_byte_labeled": 19.489162369205335,
      "perplexity_per_byte_ablated": 19.70446564237146
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.769400787353516,
      "eval_nll_ablated": 2.7761905193328857,
      "label_use_gap": 0.0067897319793699396,
      "perplexity_per_byte_labeled": 15.949074258922597,
      "perplexity_per_byte_ablated": 16.057732661742765
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.724874711036682,
      "eval_nll_ablated": 2.7364274740219114,
      "label_use_gap": 0.011552762985229226,
      "perplexity_per_byte_labeled": 15.254502586311752,
      "perplexity_per_byte_ablated": 15.431756151920634
    },
    {
      "train_per_category": 30,
      "seed": 11099,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8518800576527914,
      "eval_nll_ablated": 2.8674187819163004,
      "label_use_gap": 0.015538724263508996,
      "perplexity_per_byte_labeled": 17.320314439114696,
      "perplexity_per_byte_ablated": 17.591551913960977
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9508444627126056,
      "eval_nll_ablated": 2.9508444627126056,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.122094808054726,
      "perplexity_per_byte_ablated": 19.122094808054726
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.758986822764079,
      "eval_nll_ablated": 2.758986822764079,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.78384301380747,
      "perplexity_per_byte_ablated": 15.78384301380747
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.646624541282654,
      "eval_nll_ablated": 2.646624541282654,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 14.106342815183591,
      "perplexity_per_byte_ablated": 14.106342815183591
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.596218498547872,
      "eval_nll_ablated": 2.596218498547872,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.412921032910726,
      "perplexity_per_byte_ablated": 13.412921032910726
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9523977915445965,
      "eval_nll_ablated": 2.963498099644979,
      "label_use_gap": 0.011100308100382428,
      "perplexity_per_byte_labeled": 19.151820790383866,
      "perplexity_per_byte_ablated": 19.365596193201927
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7597039461135866,
      "eval_nll_ablated": 2.768148994445801,
      "label_use_gap": 0.008445048332214267,
      "perplexity_per_byte_labeled": 15.795166035694075,
      "perplexity_per_byte_ablated": 15.929121811652085
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.646726457277934,
      "eval_nll_ablated": 2.6583683252334596,
      "label_use_gap": 0.011641867955525687,
      "perplexity_per_byte_labeled": 14.107780550414226,
      "perplexity_per_byte_ablated": 14.272981225105873
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.59318368434906,
      "eval_nll_ablated": 2.6068454027175902,
      "label_use_gap": 0.013661718368530362,
      "perplexity_per_byte_labeled": 13.372277014429306,
      "perplexity_per_byte_ablated": 13.556218917242717
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.951333936055501,
      "eval_nll_ablated": 2.9628014087677004,
      "label_use_gap": 0.011467472712199278,
      "perplexity_per_byte_labeled": 19.131456854772825,
      "perplexity_per_byte_ablated": 19.35210905772872
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7605708996454874,
      "eval_nll_ablated": 2.7676528056462604,
      "label_use_gap": 0.007081906000772964,
      "perplexity_per_byte_labeled": 15.808865648281376,
      "perplexity_per_byte_ablated": 15.921219920399716
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.646876255671183,
      "eval_nll_ablated": 2.6589792807896933,
      "label_use_gap": 0.012103025118510136,
      "perplexity_per_byte_labeled": 14.10989403156708,
      "perplexity_per_byte_ablated": 14.281704046646185
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.590554960568746,
      "eval_nll_ablated": 2.601938263575236,
      "label_use_gap": 0.011383303006490042,
      "perplexity_per_byte_labeled": 13.33717115386545,
      "perplexity_per_byte_ablated": 13.489859615097128
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9519160906473796,
      "eval_nll_ablated": 2.961865282058716,
      "label_use_gap": 0.009949191411336322,
      "perplexity_per_byte_labeled": 19.142597562722663,
      "perplexity_per_byte_ablated": 19.334001508367248
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.760128625233968,
      "eval_nll_ablated": 2.767451826731364,
      "label_use_gap": 0.00732320149739607,
      "perplexity_per_byte_labeled": 15.801875337461743,
      "perplexity_per_byte_ablated": 15.91802041242388
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6468939065933226,
      "eval_nll_ablated": 2.657233699162801,
      "label_use_gap": 0.01033979256947859,
      "perplexity_per_byte_labeled": 14.110143086406044,
      "perplexity_per_byte_ablated": 14.256795912377125
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5915417035420734,
      "eval_nll_ablated": 2.6031323115030927,
      "label_use_gap": 0.011590607961019295,
      "perplexity_per_byte_labeled": 13.350338008868082,
      "perplexity_per_byte_ablated": 13.505976774433046
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9524047056833904,
      "eval_nll_ablated": 2.9617411295572915,
      "label_use_gap": 0.0093364238739011,
      "perplexity_per_byte_labeled": 19.151953209188747,
      "perplexity_per_byte_ablated": 19.33160129271691
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7597029209136963,
      "eval_nll_ablated": 2.7665696779886884,
      "label_use_gap": 0.006866757074992069,
      "perplexity_per_byte_labeled": 15.795149842499887,
      "perplexity_per_byte_ablated": 15.903984542503885
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6472453753153484,
      "eval_nll_ablated": 2.6576916217803954,
      "label_use_gap": 0.010446246465046993,
      "perplexity_per_byte_labeled": 14.115103231981177,
      "perplexity_per_byte_ablated": 14.263325916684051
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.593158260981242,
      "eval_nll_ablated": 2.604738958676656,
      "label_use_gap": 0.011580697695414255,
      "perplexity_per_byte_labeled": 13.371937050433736,
      "perplexity_per_byte_ablated": 13.527693554774292
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.952237335840861,
      "eval_nll_ablated": 2.9631819407145183,
      "label_use_gap": 0.010944604873657227,
      "perplexity_per_byte_labeled": 19.14874801802965,
      "perplexity_per_byte_ablated": 19.359474554778075
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.760238035519918,
      "eval_nll_ablated": 2.7670565287272137,
      "label_use_gap": 0.006818493207295884,
      "perplexity_per_byte_labeled": 15.803604319743455,
      "perplexity_per_byte_ablated": 15.911729294239976
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6465100208918253,
      "eval_nll_ablated": 2.6557249784469605,
      "label_use_gap": 0.00921495755513524,
      "perplexity_per_byte_labeled": 14.104727443789482,
      "perplexity_per_byte_ablated": 14.235302606817275
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.591765284538269,
      "eval_nll_ablated": 2.6028069416681925,
      "label_use_gap": 0.011041657129923443,
      "perplexity_per_byte_labeled": 13.353323224445955,
      "perplexity_per_byte_ablated": 13.501583051830943
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.952705176671346,
      "eval_nll_ablated": 2.9625030040740965,
      "label_use_gap": 0.009797827402750592,
      "perplexity_per_byte_labeled": 19.15770868012351,
      "perplexity_per_byte_ablated": 19.34633515907683
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7610636313756305,
      "eval_nll_ablated": 2.7677820920944214,
      "label_use_gap": 0.006718460718790897,
      "perplexity_per_byte_labeled": 15.816657097393318,
      "perplexity_per_byte_ablated": 15.923278451440831
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.647268772125244,
      "eval_nll_ablated": 2.6586447238922117,
      "label_use_gap": 0.011375951766967596,
      "perplexity_per_byte_labeled": 14.115433484231565,
      "perplexity_per_byte_ablated": 14.276926803224018
    },
    {
      "train_per_category": 100,
      "seed": 7079,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.594165293375651,
      "eval_nll_ablated": 2.6055546045303344,
      "label_use_gap": 0.011389311154683224,
      "perplexity_per_byte_labeled": 13.385409806832017,
      "perplexity_per_byte_ablated": 13.538731862995277
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9867491245269777,
      "eval_nll_ablated": 2.9867491245269777,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.82114157816483,
      "perplexity_per_byte_ablated": 19.82114157816483
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.771299123764038,
      "eval_nll_ablated": 2.771299123764038,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.979379723184621,
      "perplexity_per_byte_ablated": 15.979379723184621
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.648197547594706,
      "eval_nll_ablated": 2.648197547594706,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 14.12854964263277,
      "perplexity_per_byte_ablated": 14.12854964263277
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.592926009496053,
      "eval_nll_ablated": 2.592926009496053,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.368831758812178,
      "perplexity_per_byte_ablated": 13.368831758812178
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9791890462239583,
      "eval_nll_ablated": 2.9957158088684084,
      "label_use_gap": 0.016526762644450077,
      "perplexity_per_byte_labeled": 19.67185720756988,
      "perplexity_per_byte_ablated": 19.99967070899919
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7725221316019693,
      "eval_nll_ablated": 2.7838998238245645,
      "label_use_gap": 0.011377692222595215,
      "perplexity_per_byte_labeled": 15.998934585268724,
      "perplexity_per_byte_ablated": 16.182005023523327
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.647575529416402,
      "eval_nll_ablated": 2.660115949312846,
      "label_use_gap": 0.012540419896443833,
      "perplexity_per_byte_labeled": 14.119764160570018,
      "perplexity_per_byte_ablated": 14.297946839679947
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5907933553059896,
      "eval_nll_ablated": 2.6146435022354124,
      "label_use_gap": 0.023850146929422777,
      "perplexity_per_byte_labeled": 13.340351044297764,
      "perplexity_per_byte_ablated": 13.662344914191484
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9792479515075683,
      "eval_nll_ablated": 2.9962380568186444,
      "label_use_gap": 0.016990105311076142,
      "perplexity_per_byte_labeled": 19.67301601802752,
      "perplexity_per_byte_ablated": 20.010118223891524
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.771478525797526,
      "eval_nll_ablated": 2.7827415148417156,
      "label_use_gap": 0.011262989044189542,
      "perplexity_per_byte_labeled": 15.9822467135651,
      "perplexity_per_byte_ablated": 16.163272113087753
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.648330903053284,
      "eval_nll_ablated": 2.659333364168803,
      "label_use_gap": 0.011002461115519058,
      "perplexity_per_byte_labeled": 14.130433887483775,
      "perplexity_per_byte_ablated": 14.286761856064746
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5927943785985312,
      "eval_nll_ablated": 2.609727644920349,
      "label_use_gap": 0.016933266321817797,
      "perplexity_per_byte_labeled": 13.367072123302687,
      "perplexity_per_byte_ablated": 13.595347585577578
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.97930482228597,
      "eval_nll_ablated": 2.994662809371948,
      "label_use_gap": 0.015357987085978131,
      "perplexity_per_byte_labeled": 19.67413486957665,
      "perplexity_per_byte_ablated": 19.97862214981925
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.773853079477946,
      "eval_nll_ablated": 2.783556270599365,
      "label_use_gap": 0.009703191121419152,
      "perplexity_per_byte_labeled": 16.020242509996013,
      "perplexity_per_byte_ablated": 16.17644659836942
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.648293304443359,
      "eval_nll_ablated": 2.6625526030858357,
      "label_use_gap": 0.014259298642476548,
      "perplexity_per_byte_labeled": 14.129902612799633,
      "perplexity_per_byte_ablated": 14.332828465757254
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5922548214594525,
      "eval_nll_ablated": 2.6189614295959474,
      "label_use_gap": 0.026706608136494836,
      "perplexity_per_byte_labeled": 13.359861769483842,
      "perplexity_per_byte_ablated": 13.721465474391703
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.979076703389486,
      "eval_nll_ablated": 2.9942334493001304,
      "label_use_gap": 0.015156745910644531,
      "perplexity_per_byte_labeled": 19.669647339505588,
      "perplexity_per_byte_ablated": 19.970045968444865
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7727482318878174,
      "eval_nll_ablated": 2.7832072178522744,
      "label_use_gap": 0.010458985964457046,
      "perplexity_per_byte_labeled": 16.002552357926024,
      "perplexity_per_byte_ablated": 16.170801150582943
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.647609003384908,
      "eval_nll_ablated": 2.6600026289621987,
      "label_use_gap": 0.012393625577290734,
      "perplexity_per_byte_labeled": 14.120236813021574,
      "perplexity_per_byte_ablated": 14.296326683130628
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5909701585769653,
      "eval_nll_ablated": 2.61299622853597,
      "label_use_gap": 0.02202606995900469,
      "perplexity_per_byte_labeled": 13.342709870516309,
      "perplexity_per_byte_ablated": 13.63985781902676
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.979605484008789,
      "eval_nll_ablated": 2.997051239013672,
      "label_use_gap": 0.01744575500488299,
      "perplexity_per_byte_labeled": 19.68005101819666,
      "perplexity_per_byte_ablated": 20.026396713543527
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7733179092407227,
      "eval_nll_ablated": 2.785383852322896,
      "label_use_gap": 0.012065943082173458,
      "perplexity_per_byte_labeled": 16.011671246758628,
      "perplexity_per_byte_ablated": 16.206037408099554
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.648416550954183,
      "eval_nll_ablated": 2.661454931894938,
      "label_use_gap": 0.013038380940755268,
      "perplexity_per_byte_labeled": 14.131644181313854,
      "perplexity_per_byte_ablated": 14.317104364391659
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.591655890146891,
      "eval_nll_ablated": 2.6163474241892497,
      "label_use_gap": 0.024691534042358576,
      "perplexity_per_byte_labeled": 13.351862525676527,
      "perplexity_per_byte_ablated": 13.685644328185983
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.979938586552938,
      "eval_nll_ablated": 2.9965232531229655,
      "label_use_gap": 0.01658466657002755,
      "perplexity_per_byte_labeled": 19.686607585203756,
      "perplexity_per_byte_ablated": 20.015825849516187
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.772594356536865,
      "eval_nll_ablated": 2.784078097343445,
      "label_use_gap": 0.011483740806579767,
      "perplexity_per_byte_labeled": 16.000090149007306,
      "perplexity_per_byte_ablated": 16.184890103660464
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6494729280471803,
      "eval_nll_ablated": 2.6610708713531492,
      "label_use_gap": 0.011597943305968972,
      "perplexity_per_byte_labeled": 14.146580414271575,
      "perplexity_per_byte_ablated": 14.3116067853018
    },
    {
      "train_per_category": 100,
      "seed": 8081,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5952664534250895,
      "eval_nll_ablated": 2.612517507870992,
      "label_use_gap": 0.01725105444590236,
      "perplexity_per_byte_labeled": 13.400157403598666,
      "perplexity_per_byte_ablated": 13.633329699918862
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9701132933298746,
      "eval_nll_ablated": 2.9701132933298746,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.494128025605608,
      "perplexity_per_byte_ablated": 19.494128025605608
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.752883235613505,
      "eval_nll_ablated": 2.752883235613505,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.687798359030761,
      "perplexity_per_byte_ablated": 15.687798359030761
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6463469584782917,
      "eval_nll_ablated": 2.6463469584782917,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 14.102427680397845,
      "perplexity_per_byte_ablated": 14.102427680397845
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5918314536412557,
      "eval_nll_ablated": 2.5918314536412557,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.354206831099013,
      "perplexity_per_byte_ablated": 13.354206831099013
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9728142261505126,
      "eval_nll_ablated": 2.9779096444447837,
      "label_use_gap": 0.00509541829427107,
      "perplexity_per_byte_labeled": 19.54685152506308,
      "perplexity_per_byte_ablated": 19.646705091720623
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.754997142155965,
      "eval_nll_ablated": 2.7590946753819785,
      "label_use_gap": 0.004097533226013361,
      "perplexity_per_byte_labeled": 15.720995974584831,
      "perplexity_per_byte_ablated": 15.785545434400635
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.647512133916219,
      "eval_nll_ablated": 2.652603284517924,
      "label_use_gap": 0.005091150601705063,
      "perplexity_per_byte_labeled": 14.118869059431582,
      "perplexity_per_byte_ablated": 14.19093363829261
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5894673506418866,
      "eval_nll_ablated": 2.5971813201904297,
      "label_use_gap": 0.007713969548543087,
      "perplexity_per_byte_labeled": 13.322673399502177,
      "perplexity_per_byte_ablated": 13.425841502626936
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.974363263448079,
      "eval_nll_ablated": 2.979606103897095,
      "label_use_gap": 0.005242840449015684,
      "perplexity_per_byte_labeled": 19.57715379073603,
      "perplexity_per_byte_ablated": 19.68006321763393
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7550434350967405,
      "eval_nll_ablated": 2.7602431058883665,
      "label_use_gap": 0.0051996707916259766,
      "perplexity_per_byte_labeled": 15.721723762566004,
      "perplexity_per_byte_ablated": 15.803684450043319
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6477357864379885,
      "eval_nll_ablated": 2.652571185429891,
      "label_use_gap": 0.004835398991902551,
      "perplexity_per_byte_labeled": 14.12202713324368,
      "perplexity_per_byte_ablated": 14.19047812957523
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5907990376154584,
      "eval_nll_ablated": 2.597761313120524,
      "label_use_gap": 0.006962275505065563,
      "perplexity_per_byte_labeled": 13.34042684851619,
      "perplexity_per_byte_ablated": 13.433630654387152
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9740233580271402,
      "eval_nll_ablated": 2.9794797897338867,
      "label_use_gap": 0.0054564317067464785,
      "perplexity_per_byte_labeled": 19.570500540837916,
      "perplexity_per_byte_ablated": 19.67757750391044
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7555524508158364,
      "eval_nll_ablated": 2.7595299005508425,
      "label_use_gap": 0.003977449735006022,
      "perplexity_per_byte_labeled": 15.729728404163803,
      "perplexity_per_byte_ablated": 15.79241719635134
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6478564103444415,
      "eval_nll_ablated": 2.6526363372802733,
      "label_use_gap": 0.004779926935831735,
      "perplexity_per_byte_labeled": 14.123730690066303,
      "perplexity_per_byte_ablated": 14.191402695601447
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5900980313618978,
      "eval_nll_ablated": 2.59837064743042,
      "label_use_gap": 0.008272616068522254,
      "perplexity_per_byte_labeled": 13.331078402912453,
      "perplexity_per_byte_ablated": 13.441818720832904
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.973922602335612,
      "eval_nll_ablated": 2.9796910762786863,
      "label_use_gap": 0.005768473943074337,
      "perplexity_per_byte_labeled": 19.56852880085605,
      "perplexity_per_byte_ablated": 19.681735550525453
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.755716101328532,
      "eval_nll_ablated": 2.760660243034363,
      "label_use_gap": 0.004944141705831129,
      "perplexity_per_byte_labeled": 15.732302792925978,
      "perplexity_per_byte_ablated": 15.81027812900964
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.648606300354004,
      "eval_nll_ablated": 2.653704786300659,
      "label_use_gap": 0.005098485946655185,
      "perplexity_per_byte_labeled": 14.134325906735592,
      "perplexity_per_byte_ablated": 14.206573589129391
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.590976055463155,
      "eval_nll_ablated": 2.5989147424697876,
      "label_use_gap": 0.007938687006632694,
      "perplexity_per_byte_labeled": 13.342788551189862,
      "perplexity_per_byte_ablated": 13.449134337733945
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.974933131535848,
      "eval_nll_ablated": 2.9805521806081137,
      "label_use_gap": 0.005619049072265625,
      "perplexity_per_byte_labeled": 19.588313365371437,
      "perplexity_per_byte_ablated": 19.69869087732361
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.755521464347839,
      "eval_nll_ablated": 2.7597617467244464,
      "label_use_gap": 0.0042402823766072295,
      "perplexity_per_byte_labeled": 15.729241002989461,
      "perplexity_per_byte_ablated": 15.796079032325197
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6480170011520388,
      "eval_nll_ablated": 2.653283397356669,
      "label_use_gap": 0.005266396204630208,
      "perplexity_per_byte_labeled": 14.125999013515179,
      "perplexity_per_byte_ablated": 14.20058835723079
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.59041805267334,
      "eval_nll_ablated": 2.598993182182312,
      "label_use_gap": 0.00857512950897199,
      "perplexity_per_byte_labeled": 13.335345314820849,
      "perplexity_per_byte_ablated": 13.450189325341022
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9735031604766844,
      "eval_nll_ablated": 2.978618590037028,
      "label_use_gap": 0.0051154295603437205,
      "perplexity_per_byte_labeled": 19.560322661878743,
      "perplexity_per_byte_ablated": 19.660638475119313
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7556576251983644,
      "eval_nll_ablated": 2.759982943534851,
      "label_use_gap": 0.004325318336486639,
      "perplexity_per_byte_labeled": 15.731382855637474,
      "perplexity_per_byte_ablated": 15.799573461088048
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6481890122095746,
      "eval_nll_ablated": 2.652469833691915,
      "label_use_gap": 0.004280821482340436,
      "perplexity_per_byte_labeled": 14.128429050534871,
      "perplexity_per_byte_ablated": 14.189039972835161
    },
    {
      "train_per_category": 100,
      "seed": 9091,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5907846132914227,
      "eval_nll_ablated": 2.5986352920532227,
      "label_use_gap": 0.007850678761800012,
      "perplexity_per_byte_labeled": 13.340234423264357,
      "perplexity_per_byte_ablated": 13.445376496630415
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.990404240290324,
      "eval_nll_ablated": 2.990404240290324,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.893722710609158,
      "perplexity_per_byte_ablated": 19.893722710609158
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.774444659550985,
      "eval_nll_ablated": 2.774444659550985,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.029722570061015,
      "perplexity_per_byte_ablated": 16.029722570061015
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.648898720741272,
      "eval_nll_ablated": 2.648898720741272,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 14.138459676160785,
      "perplexity_per_byte_ablated": 14.138459676160785
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5923222382863362,
      "eval_nll_ablated": 2.5923222382863362,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.360762479333106,
      "perplexity_per_byte_ablated": 13.360762479333106
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.997851864496867,
      "eval_nll_ablated": 3.001725673675537,
      "label_use_gap": 0.0038738091786703066,
      "perplexity_per_byte_labeled": 20.04243677727404,
      "perplexity_per_byte_ablated": 20.120227929584296
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7765673796335855,
      "eval_nll_ablated": 2.7856125354766847,
      "label_use_gap": 0.009045155843099284,
      "perplexity_per_byte_labeled": 16.063785324134674,
      "perplexity_per_byte_ablated": 16.209743879632104
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6490773995717367,
      "eval_nll_ablated": 2.6580994923909507,
      "label_use_gap": 0.009022092819213956,
      "perplexity_per_byte_labeled": 14.140986145306849,
      "perplexity_per_byte_ablated": 14.269144694707844
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.588723134994507,
      "eval_nll_ablated": 2.614810260136922,
      "label_use_gap": 0.02608712514241507,
      "perplexity_per_byte_labeled": 13.31276214600576,
      "perplexity_per_byte_ablated": 13.664623408132229
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9984360694885255,
      "eval_nll_ablated": 3.002118984858195,
      "label_use_gap": 0.0036829153696693595,
      "perplexity_per_byte_labeled": 20.054149089746918,
      "perplexity_per_byte_ablated": 20.128142996666785
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.777183254559835,
      "eval_nll_ablated": 2.785978158315023,
      "label_use_gap": 0.008794903755187988,
      "perplexity_per_byte_labeled": 16.07368165387433,
      "perplexity_per_byte_ablated": 16.215671615789933
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.649221968650818,
      "eval_nll_ablated": 2.657756741841634,
      "label_use_gap": 0.008534773190815859,
      "perplexity_per_byte_labeled": 14.14303064243315,
      "perplexity_per_byte_ablated": 14.264254775584558
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5893944978713987,
      "eval_nll_ablated": 2.6089775482813518,
      "label_use_gap": 0.019583050409953007,
      "perplexity_per_byte_labeled": 13.321702841189078,
      "perplexity_per_byte_ablated": 13.585153584768475
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9985026359558105,
      "eval_nll_ablated": 3.002238416671753,
      "label_use_gap": 0.003735780715942383,
      "perplexity_per_byte_labeled": 20.05548406803813,
      "perplexity_per_byte_ablated": 20.13054708084764
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7783040603001914,
      "eval_nll_ablated": 2.7862967173258464,
      "label_use_gap": 0.007992657025654992,
      "perplexity_per_byte_labeled": 16.09170722823728,
      "perplexity_per_byte_ablated": 16.22083808696898
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6497838258743287,
      "eval_nll_ablated": 2.657697256406148,
      "label_use_gap": 0.007913430531819365,
      "perplexity_per_byte_labeled": 14.15097923914107,
      "perplexity_per_byte_ablated": 14.263406285414003
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5894798835118613,
      "eval_nll_ablated": 2.6111138582229616,
      "label_use_gap": 0.02163397471110029,
      "perplexity_per_byte_labeled": 13.322840371881927,
      "perplexity_per_byte_ablated": 13.61420670561593
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9986390113830566,
      "eval_nll_ablated": 3.0019853115081787,
      "label_use_gap": 0.003346300125122159,
      "perplexity_per_byte_labeled": 20.05821932975354,
      "perplexity_per_byte_ablated": 20.125452580185314
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.778185168902079,
      "eval_nll_ablated": 2.786531360944112,
      "label_use_gap": 0.008346192042032907,
      "perplexity_per_byte_labeled": 16.08979417639186,
      "perplexity_per_byte_ablated": 16.224644649684368
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6501237710316974,
      "eval_nll_ablated": 2.6592623472213743,
      "label_use_gap": 0.00913857618967695,
      "perplexity_per_byte_labeled": 14.155790613760857,
      "perplexity_per_byte_ablated": 14.285747289875214
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5895209550857543,
      "eval_nll_ablated": 2.6129247029622396,
      "label_use_gap": 0.023403747876485248,
      "perplexity_per_byte_labeled": 13.323387573141858,
      "perplexity_per_byte_ablated": 13.638882255259947
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.998650550842285,
      "eval_nll_ablated": 3.0025065581003827,
      "label_use_gap": 0.003856007258097538,
      "perplexity_per_byte_labeled": 20.05845079209317,
      "perplexity_per_byte_ablated": 20.135945638257102
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7779592752456663,
      "eval_nll_ablated": 2.7864923000335695,
      "label_use_gap": 0.008533024787903187,
      "perplexity_per_byte_labeled": 16.086160004438074,
      "perplexity_per_byte_ablated": 16.224010912668373
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6487098773320517,
      "eval_nll_ablated": 2.657220141092936,
      "label_use_gap": 0.008510263760884484,
      "perplexity_per_byte_labeled": 14.135789973320241,
      "perplexity_per_byte_ablated": 14.256602619052439
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.588580107688904,
      "eval_nll_ablated": 2.6121217648188275,
      "label_use_gap": 0.02354165712992362,
      "perplexity_per_byte_labeled": 13.310858193667713,
      "perplexity_per_byte_ablated": 13.627935471846794
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9991519451141357,
      "eval_nll_ablated": 3.002595647176107,
      "label_use_gap": 0.003443702061971088,
      "perplexity_per_byte_labeled": 20.068510506153274,
      "perplexity_per_byte_ablated": 20.137739610953343
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7785486300786335,
      "eval_nll_ablated": 2.786552166938782,
      "label_use_gap": 0.00800353686014832,
      "perplexity_per_byte_labeled": 16.095643254805797,
      "perplexity_per_byte_ablated": 16.22498222306623
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6495813290278116,
      "eval_nll_ablated": 2.6571072260538737,
      "label_use_gap": 0.0075258970260621005,
      "perplexity_per_byte_labeled": 14.14811400058069,
      "perplexity_per_byte_ablated": 14.25499292509185
    },
    {
      "train_per_category": 100,
      "seed": 10093,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.588888684908549,
      "eval_nll_ablated": 2.6120357354482016,
      "label_use_gap": 0.023147050539652714,
      "perplexity_per_byte_labeled": 13.314966255074689,
      "perplexity_per_byte_ablated": 13.626763119564307
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.971914275487264,
      "eval_nll_ablated": 2.971914275487264,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.52926823630161,
      "perplexity_per_byte_ablated": 19.52926823630161
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.752354876200358,
      "eval_nll_ablated": 2.752354876200358,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.679511752442291,
      "perplexity_per_byte_ablated": 15.679511752442291
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.647817055384318,
      "eval_nll_ablated": 2.647817055384318,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 14.123174862145563,
      "perplexity_per_byte_ablated": 14.123174862145563
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5875685691833494,
      "eval_nll_ablated": 2.5875685691833494,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.297400555669856,
      "perplexity_per_byte_ablated": 13.297400555669856
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9769320329030355,
      "eval_nll_ablated": 2.988020833333333,
      "label_use_gap": 0.011088800430297585,
      "perplexity_per_byte_labeled": 19.62750763142397,
      "perplexity_per_byte_ablated": 19.846364333062283
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.752559121449788,
      "eval_nll_ablated": 2.759699479738871,
      "label_use_gap": 0.007140358289082993,
      "perplexity_per_byte_labeled": 15.682714545297591,
      "perplexity_per_byte_ablated": 15.795095488721412
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6477957566579184,
      "eval_nll_ablated": 2.6580936272939044,
      "label_use_gap": 0.010297870635985973,
      "perplexity_per_byte_labeled": 14.122874059711648,
      "perplexity_per_byte_ablated": 14.269061005034867
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.591479508082072,
      "eval_nll_ablated": 2.6025468190511067,
      "label_use_gap": 0.011067310969034683,
      "perplexity_per_byte_labeled": 13.349507704275304,
      "perplexity_per_byte_ablated": 13.498071441457142
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9787917455037434,
      "eval_nll_ablated": 2.987801424662272,
      "label_use_gap": 0.009009679158528794,
      "perplexity_per_byte_labeled": 19.664043116908058,
      "perplexity_per_byte_ablated": 19.842010346307266
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7535922129948935,
      "eval_nll_ablated": 2.759542473157247,
      "label_use_gap": 0.005950260162353338,
      "perplexity_per_byte_labeled": 15.698924596890542,
      "perplexity_per_byte_ablated": 15.792615749445085
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.648875856399536,
      "eval_nll_ablated": 2.6568228960037232,
      "label_use_gap": 0.00794703960418719,
      "perplexity_per_byte_labeled": 14.13813641328274,
      "perplexity_per_byte_ablated": 14.250940378396267
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5920902490615845,
      "eval_nll_ablated": 2.5992933909098306,
      "label_use_gap": 0.0072031418482461085,
      "perplexity_per_byte_labeled": 13.357663285907073,
      "perplexity_per_byte_ablated": 13.454227795724963
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9776400566101073,
      "eval_nll_ablated": 2.986976480484009,
      "label_use_gap": 0.009336423873901545,
      "perplexity_per_byte_labeled": 19.641409292909962,
      "perplexity_per_byte_ablated": 19.82564854510126
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.753666075070699,
      "eval_nll_ablated": 2.7597965399424234,
      "label_use_gap": 0.006130464871724417,
      "perplexity_per_byte_labeled": 15.70008419487381,
      "perplexity_per_byte_ablated": 15.796628638307375
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6490151643753053,
      "eval_nll_ablated": 2.658056894938151,
      "label_use_gap": 0.009041730562845629,
      "perplexity_per_byte_labeled": 14.140106105641365,
      "perplexity_per_byte_ablated": 14.268536878436022
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5914656718571982,
      "eval_nll_ablated": 2.598643453915914,
      "label_use_gap": 0.007177782058715554,
      "perplexity_per_byte_labeled": 13.349322998762569,
      "perplexity_per_byte_ablated": 13.44548623639505
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9783087253570555,
      "eval_nll_ablated": 2.987127685546875,
      "label_use_gap": 0.008818960189819425,
      "perplexity_per_byte_labeled": 19.654547281441793,
      "perplexity_per_byte_ablated": 19.82864651018392
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7536633968353272,
      "eval_nll_ablated": 2.7592733542124432,
      "label_use_gap": 0.005609957377116004,
      "perplexity_per_byte_labeled": 15.700042146409288,
      "perplexity_per_byte_ablated": 15.788366229197756
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.648082240422567,
      "eval_nll_ablated": 2.656820201873779,
      "label_use_gap": 0.008737961451212417,
      "perplexity_per_byte_labeled": 14.126920613448227,
      "perplexity_per_byte_ablated": 14.250901984562782
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5923462470372516,
      "eval_nll_ablated": 2.5994261582692464,
      "label_use_gap": 0.0070799112319948065,
      "perplexity_per_byte_labeled": 13.361083258402248,
      "perplexity_per_byte_ablated": 13.456014196607619
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9790289878845213,
      "eval_nll_ablated": 2.987295722961426,
      "label_use_gap": 0.008266735076904563,
      "perplexity_per_byte_labeled": 19.668708814741578,
      "perplexity_per_byte_ablated": 19.831978744639727
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7546318689982097,
      "eval_nll_ablated": 2.7600831031799316,
      "label_use_gap": 0.005451234181721887,
      "perplexity_per_byte_labeled": 15.715254565397302,
      "perplexity_per_byte_ablated": 15.80115602001128
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.648878812789917,
      "eval_nll_ablated": 2.6578153053919475,
      "label_use_gap": 0.008936492602030555,
      "perplexity_per_byte_labeled": 14.138178211195022,
      "perplexity_per_byte_ablated": 14.265090165448253
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5924086570739746,
      "eval_nll_ablated": 2.5990760962168378,
      "label_use_gap": 0.006667439142863163,
      "perplexity_per_byte_labeled": 13.3619171501204,
      "perplexity_per_byte_ablated": 13.451304581037656
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9781498114267984,
      "eval_nll_ablated": 2.988640642166138,
      "label_use_gap": 0.01049083073933943,
      "perplexity_per_byte_labeled": 19.65142414824713,
      "perplexity_per_byte_ablated": 19.858669097882014
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.753691029548645,
      "eval_nll_ablated": 2.7598500490188598,
      "label_use_gap": 0.006159019470214577,
      "perplexity_per_byte_labeled": 15.70047598716707,
      "perplexity_per_byte_ablated": 15.797473923931644
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.648895748456319,
      "eval_nll_ablated": 2.6579119602839154,
      "label_use_gap": 0.009016211827596265,
      "perplexity_per_byte_labeled": 14.138417652692286,
      "perplexity_per_byte_ablated": 14.266469022832688
    },
    {
      "train_per_category": 100,
      "seed": 11099,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5930905103683473,
      "eval_nll_ablated": 2.600123167037964,
      "label_use_gap": 0.00703265666961661,
      "perplexity_per_byte_labeled": 13.371031124191857,
      "perplexity_per_byte_ablated": 13.465396425862831
    }
  ]
}