{
  "experiment": "H1 real-text conditional byte-model pilot v10 (regularised small data)",
  "completed_utc": "2026-11-18T03:58:59.819154+00:00",
  "smoke_test_not_protocol": false,
  "protocol": {
    "dataset": "SetFit/20_newsgroups",
    "revision": "f1b91292074e7cfb69be58b642d583ec262f30ed",
    "dataset_split": "train only; official test unused",
    "selected_data_sha256": "72325b82d18ed20a73b97086b6447e027c820a42c3ce4440cf75e384875095c1",
    "categories": [
      "comp.graphics",
      "comp.os.ms-windows.misc",
      "comp.sys.ibm.pc.hardware",
      "comp.sys.mac.hardware",
      "rec.autos",
      "rec.motorcycles",
      "rec.sport.baseball",
      "rec.sport.hockey"
    ],
    "sizes_train_per_category": [
      30,
      100
    ],
    "subsets_nested": true,
    "validation_per_category": 80,
    "validation_identical_across_sizes": true,
    "validation_messages": 640,
    "bytes_per_example": 96,
    "seed_values": [
      2027,
      3037,
      4049,
      5051,
      6067
    ],
    "steps": [
      100,
      200,
      400,
      800
    ],
    "total_updates": 800,
    "batch_size": 24,
    "conditions": [
      "none",
      "hierarchy",
      "scrambled_1",
      "scrambled_2",
      "scrambled_3",
      "scrambled_4",
      "scrambled_5"
    ],
    "permutation_seeds_selected": [
      31415,
      27182,
      16180,
      14142,
      17320
    ],
    "permutation_seeds_rejected": [],
    "permutation_seeds_primary": [
      31415,
      27182,
      16180,
      14142,
      17320
    ],
    "permutation_seeds_fallback": [
      11235,
      13579,
      24680,
      97531
    ],
    "scrambled_permutations": {
      "scrambled_1": [
        2,
        1,
        5,
        6,
        7,
        0,
        3,
        4
      ],
      "scrambled_2": [
        1,
        2,
        5,
        6,
        7,
        3,
        0,
        4
      ],
      "scrambled_3": [
        5,
        3,
        0,
        2,
        4,
        7,
        6,
        1
      ],
      "scrambled_4": [
        5,
        3,
        4,
        0,
        6,
        2,
        7,
        1
      ],
      "scrambled_5": [
        1,
        6,
        4,
        5,
        2,
        7,
        0,
        3
      ]
    },
    "permutation_rejection_rule": "rejected if categories 0-3 all fall in the same code//4 group (comp/rec split preserved)",
    "same_batch_sequence_per_size_and_seed_across_conditions": true,
    "batch_generator_seed": "seed + 30000, per (size, seed)",
    "evaluations": "each checkpoint: (a) eval_nll_labeled with the condition label prefix; (b) eval_nll_ablated with dummy prefix [256, 256] for all conditions; label_use_gap = ablated - labeled",
    "metric": "validation NLL in nats per byte, lower is better",
    "model": "randomly initialized 2-layer transformer width 64, 4 heads, FF 128, dropout 0.2 in training only (eval mode at every evaluation)",
    "dropout": 0.2,
    "weight_decay": 0.1,
    "learning_rate": 0.001,
    "regularisation_chosen_before_run": true,
    "v9_reference": "v9 used dropout 0 and weight decay 0.01",
    "primary_checkpoint": 800,
    "checkpoint_selection": "none; all checkpoints are reported and the primary criterion is fixed in advance at 800 updates",
    "limitations": "a single regularisation setting (dropout 0.2, weight decay 0.1), chosen once and not tuned; five seeds, five scrambled permutations, only two groups of four categories, short prefixes, tiny model; the number of updates is constant (800), so at 30 per category there are still about 80 epochs; one historical text dataset (20 Newsgroups); no significance test is run or claimed; no data-savings percentage is estimated; labels present at labeled evaluation; a null result is informative; the comparison to v9 is not matched on regularisation (v9 had dropout 0, weight decay 0.01) and sizes 30/100 only; not a pretrained-LLM or general data-savings result"
  },
  "environment": {
    "device": "cuda",
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition MIG 2g.48gb",
    "torch": "2.13.0+cu130",
    "cuda": "13.0",
    "python": "3.12.12",
    "code_sha256": "9192a0137bf7d07cec1723460e125a92313ebb334d3b6c201716ae12ac3663ff"
  },
  "summary": {
    "30": {
      "100": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.9494555473327635,
          "hierarchy": 2.9532222429911292,
          "scrambled_1": 2.9528174877166746,
          "scrambled_2": 2.9529475847880042,
          "scrambled_3": 2.9529930432637532,
          "scrambled_4": 2.9528297932942706,
          "scrambled_5": 2.9529639657338462,
          "scrambled_mean": 2.95291037495931
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": 0.00031186803181970646,
          "min": -0.000537557601929084,
          "max": 0.0009893957773847717,
          "per_seed": {
            "2027": -0.000537557601929084,
            "3037": 0.0009893957773847717,
            "4049": 0.0003491783142091087,
            "5051": 0.0005053265889487513,
            "6067": 0.0002529970804849846
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 3.8,
          "per_seed": {
            "2027": 1,
            "3037": 5,
            "4049": 5,
            "5051": 5,
            "6067": 3
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 4.8,
          "per_seed": {
            "2027": 2,
            "3037": 6,
            "4049": 6,
            "5051": 6,
            "6067": 4
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0009559377034507044,
          "per_seed": {
            "2027": 0.001304451624552616,
            "3037": 0.0009761015574141219,
            "4049": 0.0007672150929769472,
            "5051": 0.0010402202606201172,
            "6067": 0.0006916999816897196
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 1
      },
      "200": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.7750678555170696,
          "hierarchy": 2.7777817567189538,
          "scrambled_1": 2.777862695058187,
          "scrambled_2": 2.7782300853729245,
          "scrambled_3": 2.777912982304891,
          "scrambled_4": 2.7780890099207562,
          "scrambled_5": 2.778320492108663,
          "scrambled_mean": 2.778083052953084
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.000301296234130799,
          "min": -0.0007593965530396929,
          "max": 0.00035645325978617137,
          "per_seed": {
            "2027": -0.0005989678700766277,
            "3037": -2.315839131572517e-06,
            "4049": -0.0007593965530396929,
            "5051": 0.00035645325978617137,
            "6067": -0.0005022541681922732
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.4,
          "per_seed": {
            "2027": 0,
            "3037": 3,
            "4049": 0,
            "5051": 4,
            "6067": 0
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.4,
          "per_seed": {
            "2027": 1,
            "3037": 4,
            "4049": 1,
            "5051": 5,
            "6067": 1
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0011708990732829604,
          "per_seed": {
            "2027": 0.001257125536600956,
            "3037": 0.0009502728780108605,
            "4049": 0.0010248025258383642,
            "5051": 0.001774501800537287,
            "6067": 0.0008477926254273349
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 4
      },
      "400": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.71919571240743,
          "hierarchy": 2.7250887743631997,
          "scrambled_1": 2.7248121325174965,
          "scrambled_2": 2.7267331250508624,
          "scrambled_3": 2.725367781321208,
          "scrambled_4": 2.727492829958598,
          "scrambled_5": 2.7259926907221477,
          "scrambled_mean": 2.7260797119140627
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.000990937550862725,
          "min": -0.0027372137705485855,
          "max": 5.472977956122449e-05,
          "per_seed": {
            "2027": -0.001028399467468244,
            "3037": -4.257520039896434e-05,
            "4049": -0.0027372137705485855,
            "5051": 5.472977956122449e-05,
            "6067": -0.0012012290954590554
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.6,
          "per_seed": {
            "2027": 1,
            "3037": 2,
            "4049": 1,
            "5051": 2,
            "6067": 2
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.6,
          "per_seed": {
            "2027": 2,
            "3037": 3,
            "4049": 2,
            "5051": 3,
            "6067": 3
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.003333675066630004,
          "per_seed": {
            "2027": 0.0022146940231322354,
            "3037": 0.0012799580891926787,
            "4049": 0.005076201756795129,
            "5051": 0.0029217322667438594,
            "6067": 0.005175789197286118
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 4
      },
      "800": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.8208084138234457,
          "hierarchy": 2.831448631286621,
          "scrambled_1": 2.8330016803741453,
          "scrambled_2": 2.8346462694803876,
          "scrambled_3": 2.8343200079600015,
          "scrambled_4": 2.8348847325642903,
          "scrambled_5": 2.8321273644765217,
          "scrambled_mean": 2.8337960109710694
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.002347379684448203,
          "min": -0.005835297902424763,
          "max": 0.0026300843556721354,
          "per_seed": {
            "2027": -0.004605131149292063,
            "3037": 0.0026300843556721354,
            "4049": -0.00015152295430498341,
            "5051": -0.005835297902424763,
            "6067": -0.003775030771891341
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.6,
          "per_seed": {
            "2027": 0,
            "3037": 4,
            "4049": 3,
            "5051": 0,
            "6067": 1
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.6,
          "per_seed": {
            "2027": 1,
            "3037": 5,
            "4049": 4,
            "5051": 1,
            "6067": 2
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.007311223347981777,
          "per_seed": {
            "2027": 0.005033397674560458,
            "3037": 0.008080005645751953,
            "4049": 0.0068720181783041845,
            "5051": 0.006754048665364554,
            "6067": 0.009816646575927734
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 4
      }
    },
    "100": {
      "100": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.949913845062256,
          "hierarchy": 2.954889068603516,
          "scrambled_1": 2.954396149317424,
          "scrambled_2": 2.9546212069193523,
          "scrambled_3": 2.9550297991434733,
          "scrambled_4": 2.954955981572469,
          "scrambled_5": 2.9543062909444173,
          "scrambled_mean": 2.9546618855794273
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": 0.000227183024088351,
          "min": -0.00031720161438020256,
          "max": 0.0006168111165365531,
          "per_seed": {
            "2027": 0.0001243336995440103,
            "3037": -0.00031720161438020256,
            "4049": 0.0006168111165365531,
            "5051": 0.00041158994038870134,
            "6067": 0.00030038197835269287
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 3.6,
          "per_seed": {
            "2027": 5,
            "3037": 2,
            "4049": 5,
            "5051": 4,
            "6067": 2
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 4.6,
          "per_seed": {
            "2027": 6,
            "3037": 3,
            "4049": 6,
            "5051": 5,
            "6067": 3
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0011219024658203126,
          "per_seed": {
            "2027": 0.00017414093017587007,
            "3037": 0.0014734586079918621,
            "4049": 0.0014730930328368252,
            "5051": 0.001039409637451083,
            "6067": 0.001449410120645922
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 1
      },
      "200": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.7538834524154665,
          "hierarchy": 2.7557191101710004,
          "scrambled_1": 2.756126160621643,
          "scrambled_2": 2.7564629888534546,
          "scrambled_3": 2.755700187683105,
          "scrambled_4": 2.7559325440724693,
          "scrambled_5": 2.7559974257151287,
          "scrambled_mean": 2.75604386138916
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0003247512181600243,
          "min": -0.0010357427597047497,
          "max": 0.00031919638315880405,
          "per_seed": {
            "2027": -0.0007507721583048799,
            "3037": -0.00026066780090339137,
            "4049": 0.0001042302449540955,
            "5051": 0.00031919638315880405,
            "6067": -0.0010357427597047497
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.4,
          "per_seed": {
            "2027": 0,
            "3037": 1,
            "4049": 3,
            "5051": 3,
            "6067": 0
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.4,
          "per_seed": {
            "2027": 1,
            "3037": 2,
            "4049": 4,
            "5051": 4,
            "6067": 1
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.001341026624043895,
          "per_seed": {
            "2027": 0.0015094200770060517,
            "3037": 0.0009621938069663827,
            "4049": 0.0018933455149334755,
            "5051": 0.0010671059290565665,
            "6067": 0.0012730677922569988
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 3
      },
      "400": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.642921544710795,
          "hierarchy": 2.643388697306315,
          "scrambled_1": 2.6437358411153156,
          "scrambled_2": 2.6440572643280027,
          "scrambled_3": 2.6437064901987712,
          "scrambled_4": 2.6441247558593752,
          "scrambled_5": 2.6443368260065716,
          "scrambled_mean": 2.6439922355016074
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0006035381952921526,
          "min": -0.0009938875834150096,
          "max": -0.00014159520467105224,
          "per_seed": {
            "2027": -0.0009938875834150096,
            "3037": -0.0007722202936810696,
            "4049": -0.0001923704147337446,
            "5051": -0.00014159520467105224,
            "6067": -0.0009176174799598869
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 0.8,
          "per_seed": {
            "2027": 0,
            "3037": 0,
            "4049": 2,
            "5051": 2,
            "6067": 0
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 1.8,
          "per_seed": {
            "2027": 1,
            "3037": 1,
            "4049": 3,
            "5051": 3,
            "6067": 1
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.001354467074076293,
          "per_seed": {
            "2027": 0.0007547855377194601,
            "3037": 0.0016888777414956202,
            "4049": 0.0023824691772462714,
            "5051": 0.0013530254364013672,
            "6067": 0.000593177477518747
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 5
      },
      "800": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.5867646328608194,
          "hierarchy": 2.5866493431727093,
          "scrambled_1": 2.586160724957784,
          "scrambled_2": 2.586353561083476,
          "scrambled_3": 2.586857220331828,
          "scrambled_4": 2.5872437731424967,
          "scrambled_5": 2.5866805712382,
          "scrambled_mean": 2.586659170150757
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -9.826978047744461e-06,
          "min": -0.0016380198796590584,
          "max": 0.0020399188995359196,
          "per_seed": {
            "2027": -0.000411079724629726,
            "3037": -0.0007016881306967093,
            "4049": 0.0020399188995359196,
            "5051": -0.0016380198796590584,
            "6067": 0.0006617339452108517
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 2.6,
          "per_seed": {
            "2027": 2,
            "3037": 2,
            "4049": 5,
            "5051": 0,
            "6067": 4
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 3.6,
          "per_seed": {
            "2027": 3,
            "3037": 3,
            "4049": 6,
            "5051": 1,
            "6067": 5
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0019528023401894855,
          "per_seed": {
            "2027": 0.0016245206197100792,
            "3037": 0.0026028315226236387,
            "4049": 0.001374610265095999,
            "5051": 0.0022782246271768614,
            "6067": 0.0018838246663408498
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 3
      }
    }
  },
  "summary_label_use": {
    "30": {
      "100": {
        "none": 0.0,
        "hierarchy": 0.007618443171183387,
        "scrambled_1": 0.00805326779683444,
        "scrambled_2": 0.007651729583740341,
        "scrambled_3": 0.007559289932251012,
        "scrambled_4": 0.007530358632405587,
        "scrambled_5": 0.007698049545288121,
        "scrambled_mean": 0.007698539098103901
      },
      "200": {
        "none": 0.0,
        "hierarchy": 0.013395463625589965,
        "scrambled_1": 0.014073263804117886,
        "scrambled_2": 0.014171841939290353,
        "scrambled_3": 0.012338682810465507,
        "scrambled_4": 0.01173449675242102,
        "scrambled_5": 0.013053565025329662,
        "scrambled_mean": 0.013074370066324885
      },
      "400": {
        "none": 0.0,
        "hierarchy": 0.017186896006266307,
        "scrambled_1": 0.018606650034586458,
        "scrambled_2": 0.01904926300048828,
        "scrambled_3": 0.01590385913848884,
        "scrambled_4": 0.011550714174906407,
        "scrambled_5": 0.015654002825419067,
        "scrambled_mean": 0.016152897834777812
      },
      "800": {
        "none": 0.0,
        "hierarchy": 0.044840135574340766,
        "scrambled_1": 0.04084433873494468,
        "scrambled_2": 0.04603624661763508,
        "scrambled_3": 0.04094600359598779,
        "scrambled_4": 0.03416631380716959,
        "scrambled_5": 0.037256005605062016,
        "scrambled_mean": 0.03984978167215983
      }
    },
    "100": {
      "100": {
        "none": 0.0,
        "hierarchy": 0.007321971257527693,
        "scrambled_1": 0.007679656346639074,
        "scrambled_2": 0.00759295781453444,
        "scrambled_3": 0.007562227249145436,
        "scrambled_4": 0.00751501719156904,
        "scrambled_5": 0.007471208572387677,
        "scrambled_mean": 0.007564213434855134
      },
      "200": {
        "none": 0.0,
        "hierarchy": 0.014375262260436904,
        "scrambled_1": 0.014521800676981566,
        "scrambled_2": 0.014188831647237166,
        "scrambled_3": 0.013816823959350622,
        "scrambled_4": 0.013244552612304793,
        "scrambled_5": 0.013336702982584558,
        "scrambled_mean": 0.013821742375691741
      },
      "400": {
        "none": 0.0,
        "hierarchy": 0.022020913759867346,
        "scrambled_1": 0.021442869504292705,
        "scrambled_2": 0.021256519953409825,
        "scrambled_3": 0.020126748085021972,
        "scrambled_4": 0.01897823492685955,
        "scrambled_5": 0.020171960194905657,
        "scrambled_mean": 0.020395266532897943
      },
      "800": {
        "none": 0.0,
        "hierarchy": 0.025101928710937572,
        "scrambled_1": 0.022671422958374078,
        "scrambled_2": 0.02236858685811356,
        "scrambled_3": 0.02025406996409087,
        "scrambled_4": 0.01830229600270581,
        "scrambled_5": 0.020377427736918106,
        "scrambled_mean": 0.020794760704040485
      }
    }
  },
  "elapsed_seconds": 243.52759178500128,
  "runs": [
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.953571891784668,
      "eval_nll_ablated": 2.953571891784668,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.17432015344081,
      "perplexity_per_byte_ablated": 19.17432015344081
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7713655471801757,
      "eval_nll_ablated": 2.7713655471801757,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.980441163425448,
      "perplexity_per_byte_ablated": 15.980441163425448
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7205170075098675,
      "eval_nll_ablated": 2.7205170075098675,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.188172614731945,
      "perplexity_per_byte_ablated": 15.188172614731945
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.815731239318848,
      "eval_nll_ablated": 2.815731239318848,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.705386950918243,
      "perplexity_per_byte_ablated": 16.705386950918243
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9604031244913735,
      "eval_nll_ablated": 2.963308556874593,
      "label_use_gap": 0.0029054323832196083,
      "perplexity_per_byte_labeled": 19.305752808812443,
      "perplexity_per_byte_ablated": 19.361925932295996
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.775722114245097,
      "eval_nll_ablated": 2.782948652903239,
      "label_use_gap": 0.007226538658141823,
      "perplexity_per_byte_labeled": 16.050212899350623,
      "perplexity_per_byte_ablated": 16.16662048871718
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.732511353492737,
      "eval_nll_ablated": 2.7420296827952066,
      "label_use_gap": 0.009518329302469652,
      "perplexity_per_byte_labeled": 15.371441711244389,
      "perplexity_per_byte_ablated": 15.518450685319726
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.828744109471639,
      "eval_nll_ablated": 2.84514848391215,
      "label_use_gap": 0.016404374440511127,
      "perplexity_per_byte_labeled": 16.92419253890747,
      "perplexity_per_byte_ablated": 17.204113013286506
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9602715333302814,
      "eval_nll_ablated": 2.962953042984009,
      "label_use_gap": 0.0026815096537275984,
      "perplexity_per_byte_labeled": 19.303212509528702,
      "perplexity_per_byte_ablated": 19.35504372211175
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7764370441436768,
      "eval_nll_ablated": 2.782120196024577,
      "label_use_gap": 0.0056831518809001835,
      "perplexity_per_byte_labeled": 16.06169177923923,
      "perplexity_per_byte_ablated": 16.153232687142452
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7337724606196088,
      "eval_nll_ablated": 2.7436490217844645,
      "label_use_gap": 0.00987656116485569,
      "perplexity_per_byte_labeled": 15.390838974379355,
      "perplexity_per_byte_ablated": 15.543600675251161
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8341529528299967,
      "eval_nll_ablated": 2.8597679932912192,
      "label_use_gap": 0.025615040461222538,
      "perplexity_per_byte_labeled": 17.015980855854714,
      "perplexity_per_byte_ablated": 17.4574762151006
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9606265385945636,
      "eval_nll_ablated": 2.963119141260783,
      "label_use_gap": 0.0024926026662193124,
      "perplexity_per_byte_labeled": 19.31006646811085,
      "perplexity_per_byte_ablated": 19.3582588285253
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.777064800262451,
      "eval_nll_ablated": 2.7825945297876995,
      "label_use_gap": 0.005529729525248328,
      "perplexity_per_byte_labeled": 16.07177776997149,
      "perplexity_per_byte_ablated": 16.160896528255144
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7344776074091595,
      "eval_nll_ablated": 2.744376373291016,
      "label_use_gap": 0.009898765881856342,
      "perplexity_per_byte_labeled": 15.401695602378965,
      "perplexity_per_byte_ablated": 15.554910449211492
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.834592819213867,
      "eval_nll_ablated": 2.858833424250285,
      "label_use_gap": 0.024240605036417673,
      "perplexity_per_byte_labeled": 17.023467260209888,
      "perplexity_per_byte_ablated": 17.441168619770927
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9612508614857993,
      "eval_nll_ablated": 2.9635469118754068,
      "label_use_gap": 0.0022960503896074513,
      "perplexity_per_byte_labeled": 19.322125948751374,
      "perplexity_per_byte_ablated": 19.36654149421662
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.77580767472585,
      "eval_nll_ablated": 2.7818827946980793,
      "label_use_gap": 0.006075119972229093,
      "perplexity_per_byte_labeled": 16.05158622203272,
      "perplexity_per_byte_ablated": 16.149398343432452
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7329915444056194,
      "eval_nll_ablated": 2.7410858472188315,
      "label_use_gap": 0.008094302813212106,
      "perplexity_per_byte_labeled": 15.378824710354705,
      "perplexity_per_byte_ablated": 15.503810729415113
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.832459894816081,
      "eval_nll_ablated": 2.8492159048716226,
      "label_use_gap": 0.016756010055541726,
      "perplexity_per_byte_labeled": 16.98719618703499,
      "perplexity_per_byte_ablated": 17.274231887966124
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.961575984954834,
      "eval_nll_ablated": 2.963976192474365,
      "label_use_gap": 0.0024002075195310724,
      "perplexity_per_byte_labeled": 19.328409046704913,
      "perplexity_per_byte_ablated": 19.374856959455258
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7761706193288167,
      "eval_nll_ablated": 2.7819873332977294,
      "label_use_gap": 0.005816713968912701,
      "perplexity_per_byte_labeled": 16.05741311597695,
      "perplexity_per_byte_ablated": 16.151086667166428
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7341942389806113,
      "eval_nll_ablated": 2.742132314046224,
      "label_use_gap": 0.007938075065612615,
      "perplexity_per_byte_labeled": 15.397331866400844,
      "perplexity_per_byte_ablated": 15.520043445059475
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8352869669596354,
      "eval_nll_ablated": 2.8545610109965005,
      "label_use_gap": 0.019274044036865057,
      "perplexity_per_byte_labeled": 17.03528816388589,
      "perplexity_per_byte_ablated": 17.366811694660743
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9609784921010336,
      "eval_nll_ablated": 2.9636637846628826,
      "label_use_gap": 0.0026852925618490175,
      "perplexity_per_byte_labeled": 19.316863909836027,
      "perplexity_per_byte_ablated": 19.36880504817617
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7761252721150718,
      "eval_nll_ablated": 2.782262357076009,
      "label_use_gap": 0.0061370849609372335,
      "perplexity_per_byte_labeled": 16.056684973541927,
      "perplexity_per_byte_ablated": 16.155529210919532
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7322629133860272,
      "eval_nll_ablated": 2.740984908739726,
      "label_use_gap": 0.008721995353698553,
      "perplexity_per_byte_labeled": 15.367623302967884,
      "perplexity_per_byte_ablated": 15.502245877317966
    },
    {
      "train_per_category": 30,
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.830253569285075,
      "eval_nll_ablated": 2.854126294453939,
      "label_use_gap": 0.023872725168863962,
      "perplexity_per_byte_labeled": 16.949758217748,
      "perplexity_per_byte_ablated": 17.35926369506453
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9388124148050943,
      "eval_nll_ablated": 2.9388124148050943,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 18.893395467009853,
      "perplexity_per_byte_ablated": 18.893395467009853
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.765377648671468,
      "eval_nll_ablated": 2.765377648671468,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.885037821435594,
      "perplexity_per_byte_ablated": 15.885037821435594
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7038360834121704,
      "eval_nll_ablated": 2.7038360834121704,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 14.936921238661585,
      "perplexity_per_byte_ablated": 14.936921238661585
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8136093616485596,
      "eval_nll_ablated": 2.8136093616485596,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.66997774366209,
      "perplexity_per_byte_ablated": 16.66997774366209
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.941827726364136,
      "eval_nll_ablated": 2.9467766920725507,
      "label_use_gap": 0.004948965708414743,
      "perplexity_per_byte_labeled": 18.950450917501257,
      "perplexity_per_byte_ablated": 19.04446850226059
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7673442999521893,
      "eval_nll_ablated": 2.7724136829376222,
      "label_use_gap": 0.005069382985432913,
      "perplexity_per_byte_labeled": 15.916308890976937,
      "perplexity_per_byte_ablated": 15.997199616261215
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.712231691678365,
      "eval_nll_ablated": 2.718116052945455,
      "label_use_gap": 0.005884361267090021,
      "perplexity_per_byte_labeled": 15.062853678091326,
      "perplexity_per_byte_ablated": 15.151750244092682
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8271724224090575,
      "eval_nll_ablated": 2.8476446946461995,
      "label_use_gap": 0.020472272237141986,
      "perplexity_per_byte_labeled": 16.897613896574796,
      "perplexity_per_byte_ablated": 17.247111749486166
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.940364058812459,
      "eval_nll_ablated": 2.944875796635946,
      "label_use_gap": 0.004511737823486683,
      "perplexity_per_byte_labeled": 18.922734046494238,
      "perplexity_per_byte_ablated": 19.008301344874834
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.766826629638672,
      "eval_nll_ablated": 2.7719271024068197,
      "label_use_gap": 0.005100472768147846,
      "perplexity_per_byte_labeled": 15.908071622641899,
      "perplexity_per_byte_ablated": 15.989417583826857
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7115654786427816,
      "eval_nll_ablated": 2.7163609743118284,
      "label_use_gap": 0.004795495669046801,
      "perplexity_per_byte_labeled": 15.052821950622743,
      "perplexity_per_byte_ablated": 15.125181053306852
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.829065195719401,
      "eval_nll_ablated": 2.850236988067627,
      "label_use_gap": 0.02117179234822597,
      "perplexity_per_byte_labeled": 16.929627536891054,
      "perplexity_per_byte_ablated": 17.291879324087766
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9405734380086264,
      "eval_nll_ablated": 2.9455122470855715,
      "label_use_gap": 0.004938809076945105,
      "perplexity_per_byte_labeled": 18.926696488150128,
      "perplexity_per_byte_ablated": 19.020403037467744
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7671204010645547,
      "eval_nll_ablated": 2.772797115643819,
      "label_use_gap": 0.005676714579264441,
      "perplexity_per_byte_labeled": 15.912745646039173,
      "perplexity_per_byte_ablated": 16.003334641911227
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7125287294387816,
      "eval_nll_ablated": 2.7181671619415284,
      "label_use_gap": 0.00563843250274676,
      "perplexity_per_byte_labeled": 15.067328578987718,
      "perplexity_per_byte_ablated": 15.152524654625918
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8253188927968345,
      "eval_nll_ablated": 2.8583240350087484,
      "label_use_gap": 0.033005142211913974,
      "perplexity_per_byte_labeled": 16.866322677400856,
      "perplexity_per_byte_ablated": 17.432286538526565
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9413118680318195,
      "eval_nll_ablated": 2.9466890970865887,
      "label_use_gap": 0.005377229054769117,
      "perplexity_per_byte_labeled": 18.94067769051133,
      "perplexity_per_byte_ablated": 19.04280037537032
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7676659266153973,
      "eval_nll_ablated": 2.772877311706543,
      "label_use_gap": 0.005211385091145715,
      "perplexity_per_byte_labeled": 15.921428823605419,
      "perplexity_per_byte_ablated": 16.00461809780333
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.712012028694153,
      "eval_nll_ablated": 2.7199134985605875,
      "label_use_gap": 0.007901469866434585,
      "perplexity_per_byte_labeled": 15.059545290080143,
      "perplexity_per_byte_ablated": 15.179009182020698
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8247642199198406,
      "eval_nll_ablated": 2.8495710372924803,
      "label_use_gap": 0.024806817372639678,
      "perplexity_per_byte_labeled": 16.85696997976072,
      "perplexity_per_byte_ablated": 17.280367617188656
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9406021277109784,
      "eval_nll_ablated": 2.9454665184020996,
      "label_use_gap": 0.004864390691121212,
      "perplexity_per_byte_labeled": 18.92723949722823,
      "perplexity_per_byte_ablated": 19.019533279364335
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.767343219121297,
      "eval_nll_ablated": 2.7717574993769327,
      "label_use_gap": 0.0044142802556357275,
      "perplexity_per_byte_labeled": 15.916291688147892,
      "perplexity_per_byte_ablated": 15.98670596011481
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7124196608861286,
      "eval_nll_ablated": 2.71823894182841,
      "label_use_gap": 0.005819280942281235,
      "perplexity_per_byte_labeled": 15.06568529688409,
      "perplexity_per_byte_ablated": 15.1536123401681
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.822578191757202,
      "eval_nll_ablated": 2.8530638853708905,
      "label_use_gap": 0.03048569361368836,
      "perplexity_per_byte_labeled": 16.820160416677535,
      "perplexity_per_byte_ablated": 17.340830848985195
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9413401603698732,
      "eval_nll_ablated": 2.946955649058024,
      "label_use_gap": 0.0056154886881509825,
      "perplexity_per_byte_labeled": 18.94121357414818,
      "perplexity_per_byte_ablated": 19.04787694790718
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7677769025166827,
      "eval_nll_ablated": 2.7730263074239097,
      "label_use_gap": 0.005249404907226918,
      "perplexity_per_byte_labeled": 15.923195816563874,
      "perplexity_per_byte_ablated": 16.007002895015862
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7128454367319743,
      "eval_nll_ablated": 2.71880091826121,
      "label_use_gap": 0.005955481529235751,
      "perplexity_per_byte_labeled": 15.072101267570362,
      "perplexity_per_byte_ablated": 15.162130706511443
    },
    {
      "train_per_category": 30,
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.820985190073649,
      "eval_nll_ablated": 2.840591637293498,
      "label_use_gap": 0.01960644721984872,
      "perplexity_per_byte_labeled": 16.793387203364837,
      "perplexity_per_byte_ablated": 17.125894858491083
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.964134931564331,
      "eval_nll_ablated": 2.964134931564331,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.37793275073492,
      "perplexity_per_byte_ablated": 19.37793275073492
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7867502848307293,
      "eval_nll_ablated": 2.7867502848307293,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.228197000783126,
      "perplexity_per_byte_ablated": 16.228197000783126
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7334638992945353,
      "eval_nll_ablated": 2.7334638992945353,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.386090689317665,
      "perplexity_per_byte_ablated": 15.386090689317665
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8260221322377523,
      "eval_nll_ablated": 2.8260221322377523,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.878187912292436,
      "perplexity_per_byte_ablated": 16.878187912292436
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9657832463582356,
      "eval_nll_ablated": 2.9664505322774253,
      "label_use_gap": 0.0006672859191896308,
      "perplexity_per_byte_labeled": 19.409900022789202,
      "perplexity_per_byte_ablated": 19.42285629805656
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.787367844581604,
      "eval_nll_ablated": 2.7905018091201783,
      "label_use_gap": 0.0031339645385743964,
      "perplexity_per_byte_labeled": 16.238221977272456,
      "perplexity_per_byte_ablated": 16.289191816234975
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7335663080215453,
      "eval_nll_ablated": 2.7420454104741414,
      "label_use_gap": 0.008479102452596088,
      "perplexity_per_byte_labeled": 15.387666439962752,
      "perplexity_per_byte_ablated": 15.518694756449001
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8432717005411785,
      "eval_nll_ablated": 2.88112374941508,
      "label_use_gap": 0.03785204887390137,
      "perplexity_per_byte_labeled": 17.171854900284494,
      "perplexity_per_byte_ablated": 17.834303210938007
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9655353387196857,
      "eval_nll_ablated": 2.9664851983388263,
      "label_use_gap": 0.000949859619140625,
      "perplexity_per_byte_labeled": 19.405088756709507,
      "perplexity_per_byte_ablated": 19.423529623656275
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7884661833445232,
      "eval_nll_ablated": 2.7918192545572915,
      "label_use_gap": 0.0033530712127682882,
      "perplexity_per_byte_labeled": 16.256066843971464,
      "perplexity_per_byte_ablated": 16.31066608014703
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7333744605382284,
      "eval_nll_ablated": 2.74375467300415,
      "label_use_gap": 0.010380212465921801,
      "perplexity_per_byte_labeled": 15.384714638039064,
      "perplexity_per_byte_ablated": 15.545242962374102
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8415720462799072,
      "eval_nll_ablated": 2.8669750849405924,
      "label_use_gap": 0.025403038660685162,
      "perplexity_per_byte_labeled": 17.14269347312145,
      "perplexity_per_byte_ablated": 17.58374832692075
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.965442927678426,
      "eval_nll_ablated": 2.9663097540537517,
      "label_use_gap": 0.0008668263753257577,
      "perplexity_per_byte_labeled": 19.403295595107004,
      "perplexity_per_byte_ablated": 19.420122175305238
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.788316098848979,
      "eval_nll_ablated": 2.7921144644419353,
      "label_use_gap": 0.003798365592956543,
      "perplexity_per_byte_labeled": 16.253627243457334,
      "perplexity_per_byte_ablated": 16.315481860797068
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.737312157948812,
      "eval_nll_ablated": 2.7483584960301717,
      "label_use_gap": 0.011046338081359686,
      "perplexity_per_byte_labeled": 15.44541441928607,
      "perplexity_per_byte_ablated": 15.616975505136747
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.847758134206136,
      "eval_nll_ablated": 2.8889296213785807,
      "label_use_gap": 0.041171487172444454,
      "perplexity_per_byte_labeled": 17.249068365229924,
      "perplexity_per_byte_ablated": 17.974060251508465
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.965763521194458,
      "eval_nll_ablated": 2.9663296858469645,
      "label_use_gap": 0.0005661646525063624,
      "perplexity_per_byte_labeled": 19.409517163108344,
      "perplexity_per_byte_ablated": 19.420509257022207
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.787441380818685,
      "eval_nll_ablated": 2.7904337644577026,
      "label_use_gap": 0.0029923836390177705,
      "perplexity_per_byte_labeled": 16.23941611891935,
      "perplexity_per_byte_ablated": 16.28808346138508
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7356136083602904,
      "eval_nll_ablated": 2.7440683841705322,
      "label_use_gap": 0.008454775810241788,
      "perplexity_per_byte_labeled": 15.419201884926457,
      "perplexity_per_byte_ablated": 15.550120443695695
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8427974065144856,
      "eval_nll_ablated": 2.8792405128479004,
      "label_use_gap": 0.03644310633341474,
      "perplexity_per_byte_labeled": 17.163712323218526,
      "perplexity_per_byte_ablated": 17.800748604528902
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9654322465260825,
      "eval_nll_ablated": 2.96614351272583,
      "label_use_gap": 0.0007112661997474845,
      "perplexity_per_byte_labeled": 19.403088346657615,
      "perplexity_per_byte_ablated": 19.41689401674053
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7880183378855388,
      "eval_nll_ablated": 2.79164187113444,
      "label_use_gap": 0.0036235332489011896,
      "perplexity_per_byte_labeled": 16.248788268214646,
      "perplexity_per_byte_ablated": 16.30777309496014
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7384506622950235,
      "eval_nll_ablated": 2.7462391455968223,
      "label_use_gap": 0.00778848330179871,
      "perplexity_per_byte_labeled": 15.463009104643833,
      "perplexity_per_byte_ablated": 15.583912709532955
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.844102414449056,
      "eval_nll_ablated": 2.8649837176005044,
      "label_use_gap": 0.020881303151448538,
      "perplexity_per_byte_labeled": 17.186125725640085,
      "perplexity_per_byte_ablated": 17.548767466217804
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9649963061014812,
      "eval_nll_ablated": 2.965893840789795,
      "label_use_gap": 0.0008975346883137725,
      "perplexity_per_byte_labeled": 19.394631599538094,
      "perplexity_per_byte_ablated": 19.41204676835571
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7883942047754924,
      "eval_nll_ablated": 2.7917190074920653,
      "label_use_gap": 0.003324802716572872,
      "perplexity_per_byte_labeled": 16.2548967976516,
      "perplexity_per_byte_ablated": 16.30903106569468
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.736766719818115,
      "eval_nll_ablated": 2.7455296675364176,
      "label_use_gap": 0.008762947718302438,
      "perplexity_per_byte_labeled": 15.436992198426381,
      "perplexity_per_byte_ablated": 15.572860186595163
    },
    {
      "train_per_category": 30,
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.840886116027832,
      "eval_nll_ablated": 2.86759467124939,
      "label_use_gap": 0.026708555221557795,
      "perplexity_per_byte_labeled": 17.130938812965567,
      "perplexity_per_byte_ablated": 17.594646352426516
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9503867467244467,
      "eval_nll_ablated": 2.9503867467244467,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.113344322305377,
      "perplexity_per_byte_ablated": 19.113344322305377
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7711132367451987,
      "eval_nll_ablated": 2.7711132367451987,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.976409639983485,
      "perplexity_per_byte_ablated": 15.976409639983485
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.710487429300944,
      "eval_nll_ablated": 2.710487429300944,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.036603009806106,
      "perplexity_per_byte_ablated": 15.036603009806106
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8222465356191,
      "eval_nll_ablated": 2.8222465356191,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.814582832202667,
      "perplexity_per_byte_ablated": 16.814582832202667
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9529230912526447,
      "eval_nll_ablated": 2.957513189315796,
      "label_use_gap": 0.004590098063151249,
      "perplexity_per_byte_labeled": 19.161883879091025,
      "perplexity_per_byte_ablated": 19.250040975247163
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.773118241628011,
      "eval_nll_ablated": 2.778580594062805,
      "label_use_gap": 0.005462352434793871,
      "perplexity_per_byte_labeled": 16.008474553733915,
      "perplexity_per_byte_ablated": 16.096157743914542
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7122754017512003,
      "eval_nll_ablated": 2.7231855074564617,
      "label_use_gap": 0.010910105705261408,
      "perplexity_per_byte_labeled": 15.063512090912235,
      "perplexity_per_byte_ablated": 15.228756377344691
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.822998380661011,
      "eval_nll_ablated": 2.8670845508575438,
      "label_use_gap": 0.04408617019653294,
      "perplexity_per_byte_labeled": 16.827229546525853,
      "perplexity_per_byte_ablated": 17.585673253409798
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9526209513346355,
      "eval_nll_ablated": 2.95762456258138,
      "label_use_gap": 0.005003611246744644,
      "perplexity_per_byte_labeled": 19.156095183608922,
      "perplexity_per_byte_ablated": 19.252185034566434
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7717700481414793,
      "eval_nll_ablated": 2.7789013544718424,
      "label_use_gap": 0.007131306330363074,
      "perplexity_per_byte_labeled": 15.986906574782509,
      "perplexity_per_byte_ablated": 16.101321582189573
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.710433665911357,
      "eval_nll_ablated": 2.723582053184509,
      "label_use_gap": 0.013148387273152373,
      "perplexity_per_byte_labeled": 15.035794612791694,
      "perplexity_per_byte_ablated": 15.234796473137546
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.82531689008077,
      "eval_nll_ablated": 2.850041135152181,
      "label_use_gap": 0.024724245071411133,
      "perplexity_per_byte_labeled": 16.866288898979306,
      "perplexity_per_byte_ablated": 17.288492990731058
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9528252442677814,
      "eval_nll_ablated": 2.957871611913045,
      "label_use_gap": 0.005046367645263672,
      "perplexity_per_byte_labeled": 19.16000903825441,
      "perplexity_per_byte_ablated": 19.256941861573583
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7730913956960044,
      "eval_nll_ablated": 2.781347560882568,
      "label_use_gap": 0.008256165186563802,
      "perplexity_per_byte_labeled": 16.00804479708315,
      "perplexity_per_byte_ablated": 16.140756952127603
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7124211390813193,
      "eval_nll_ablated": 2.728111759821574,
      "label_use_gap": 0.015690620740254513,
      "perplexity_per_byte_labeled": 15.0657075669241,
      "perplexity_per_byte_ablated": 15.303962163716925
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8290482997894286,
      "eval_nll_ablated": 2.8677667140960694,
      "label_use_gap": 0.0387184143066408,
      "perplexity_per_byte_labeled": 16.92934149750619,
      "perplexity_per_byte_ablated": 17.597673645875933
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.952035109202067,
      "eval_nll_ablated": 2.956643835703532,
      "label_use_gap": 0.004608726501464933,
      "perplexity_per_byte_labeled": 19.14487602260435,
      "perplexity_per_byte_ablated": 19.23331315483819
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.77289559841156,
      "eval_nll_ablated": 2.7783310492833455,
      "label_use_gap": 0.005435450871785541,
      "perplexity_per_byte_labeled": 16.00491077220941,
      "perplexity_per_byte_ablated": 16.092141532913278
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7119455416997273,
      "eval_nll_ablated": 2.722615432739258,
      "label_use_gap": 0.010669891039530643,
      "perplexity_per_byte_labeled": 15.058544059461164,
      "perplexity_per_byte_ablated": 15.220077322449441
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8320709387461345,
      "eval_nll_ablated": 2.878079350789388,
      "label_use_gap": 0.046008412043253344,
      "perplexity_per_byte_labeled": 16.98059019876957,
      "perplexity_per_byte_ablated": 17.78009104634364
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9517850240071613,
      "eval_nll_ablated": 2.956568225224813,
      "label_use_gap": 0.004783201217651634,
      "perplexity_per_byte_labeled": 19.140088771188097,
      "perplexity_per_byte_ablated": 19.231858969799706
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.772507349650065,
      "eval_nll_ablated": 2.77867214679718,
      "label_use_gap": 0.006164797147115042,
      "perplexity_per_byte_labeled": 15.99869809153509,
      "perplexity_per_byte_ablated": 16.097631458629206
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.712947614987691,
      "eval_nll_ablated": 2.722781038284302,
      "label_use_gap": 0.009833423296610633,
      "perplexity_per_byte_labeled": 15.073641387269904,
      "perplexity_per_byte_ablated": 15.222598060368362
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.827738126118978,
      "eval_nll_ablated": 2.8635225931803387,
      "label_use_gap": 0.035784467061360736,
      "perplexity_per_byte_labeled": 16.90717564374782,
      "perplexity_per_byte_ablated": 17.523145256705483
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9528224945068358,
      "eval_nll_ablated": 2.9578625996907553,
      "label_use_gap": 0.005040105183919508,
      "perplexity_per_byte_labeled": 19.15995635288228,
      "perplexity_per_byte_ablated": 19.25676831451493
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7735445499420166,
      "eval_nll_ablated": 2.7812913417816163,
      "label_use_gap": 0.007746791839599698,
      "perplexity_per_byte_labeled": 16.01530055441776,
      "perplexity_per_byte_ablated": 16.139849558789727
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7133553981781007,
      "eval_nll_ablated": 2.724581972757975,
      "label_use_gap": 0.011226574579874526,
      "perplexity_per_byte_labeled": 15.07978941829255,
      "perplexity_per_byte_ablated": 15.250037663040965
    },
    {
      "train_per_category": 30,
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8299941380818683,
      "eval_nll_ablated": 2.870008691151937,
      "label_use_gap": 0.040014553070068626,
      "perplexity_per_byte_labeled": 16.9453614919281,
      "perplexity_per_byte_ablated": 17.637171486508326
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9403717517852783,
      "eval_nll_ablated": 2.9403717517852783,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 18.922879619132864,
      "perplexity_per_byte_ablated": 18.922879619132864
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7807325601577757,
      "eval_nll_ablated": 2.7807325601577757,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.130833426703784,
      "perplexity_per_byte_ablated": 16.130833426703784
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.727674142519633,
      "eval_nll_ablated": 2.727674142519633,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.297266350294594,
      "perplexity_per_byte_ablated": 15.297266350294594
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.826432800292969,
      "eval_nll_ablated": 2.826432800292969,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.885120668331243,
      "perplexity_per_byte_ablated": 16.885120668331243
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9451740264892576,
      "eval_nll_ablated": 2.9701544602711993,
      "label_use_gap": 0.024980433781941702,
      "perplexity_per_byte_labeled": 19.01397103318976,
      "perplexity_per_byte_ablated": 19.494930555748958
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.785356283187866,
      "eval_nll_ablated": 2.831441362698873,
      "label_use_gap": 0.04608507951100682,
      "perplexity_per_byte_labeled": 16.20559062782465,
      "perplexity_per_byte_ablated": 16.969902990470246
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7348591168721517,
      "eval_nll_ablated": 2.786001698176066,
      "label_use_gap": 0.05114258130391436,
      "perplexity_per_byte_labeled": 15.407572615994267,
      "perplexity_per_byte_ablated": 16.21605333493928
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8350565433502197,
      "eval_nll_ablated": 2.940442355473836,
      "label_use_gap": 0.10538581212361642,
      "perplexity_per_byte_labeled": 17.03136328350965,
      "perplexity_per_byte_ablated": 18.92421569139737
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.945295556386312,
      "eval_nll_ablated": 2.9724151770273846,
      "label_use_gap": 0.02711962064107265,
      "perplexity_per_byte_labeled": 19.016281939551277,
      "perplexity_per_byte_ablated": 19.53905292721739
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.785813570022583,
      "eval_nll_ablated": 2.834911886850993,
      "label_use_gap": 0.04909831682841004,
      "perplexity_per_byte_labeled": 16.213002925711518,
      "perplexity_per_byte_ablated": 17.028899764306324
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7349145968755084,
      "eval_nll_ablated": 2.789747190475464,
      "label_use_gap": 0.054832593599955626,
      "perplexity_per_byte_labeled": 15.408427451887652,
      "perplexity_per_byte_ablated": 16.276904325151143
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8349013169606527,
      "eval_nll_ablated": 2.9422088940938314,
      "label_use_gap": 0.10730757713317862,
      "perplexity_per_byte_labeled": 17.028719771654455,
      "perplexity_per_byte_ablated": 18.957675594673614
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.945269775390625,
      "eval_nll_ablated": 2.970183817545573,
      "label_use_gap": 0.024914042154947857,
      "perplexity_per_byte_labeled": 19.01579168718824,
      "perplexity_per_byte_ablated": 19.49550288217511
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.785557730992635,
      "eval_nll_ablated": 2.833155965805054,
      "label_use_gap": 0.04759823481241865,
      "perplexity_per_byte_labeled": 16.208855537324997,
      "perplexity_per_byte_ablated": 16.999024597713948
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.736925991376241,
      "eval_nll_ablated": 2.789898149172465,
      "label_use_gap": 0.052972157796224106,
      "perplexity_per_byte_labeled": 15.439451068035398,
      "perplexity_per_byte_ablated": 16.279361650891957
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8365132013956704,
      "eval_nll_ablated": 2.929558785756429,
      "label_use_gap": 0.0930455843607585,
      "perplexity_per_byte_labeled": 17.05619023365998,
      "perplexity_per_byte_ablated": 18.719369421935383
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.944603856404622,
      "eval_nll_ablated": 2.9695521354675294,
      "label_use_gap": 0.024948279062907197,
      "perplexity_per_byte_labeled": 19.003132925792432,
      "perplexity_per_byte_ablated": 19.48319181115282
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.785754330952962,
      "eval_nll_ablated": 2.8277334054311116,
      "label_use_gap": 0.041979074478149414,
      "perplexity_per_byte_labeled": 16.21204251094976,
      "perplexity_per_byte_ablated": 16.907095830437292
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7342761834462483,
      "eval_nll_ablated": 2.7786750396092734,
      "label_use_gap": 0.04439885616302508,
      "perplexity_per_byte_labeled": 15.39859364423002,
      "perplexity_per_byte_ablated": 16.09767802611952
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.839507579803467,
      "eval_nll_ablated": 2.9202232519785563,
      "label_use_gap": 0.08071567217508946,
      "perplexity_per_byte_labeled": 17.107339463271,
      "perplexity_per_byte_ablated": 18.545427300953914
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.944753583272298,
      "eval_nll_ablated": 2.9696463108062745,
      "label_use_gap": 0.024892727533976533,
      "perplexity_per_byte_labeled": 19.005978418379467,
      "perplexity_per_byte_ablated": 19.485026733742345
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7864055236180625,
      "eval_nll_ablated": 2.825058682759603,
      "label_use_gap": 0.03865315914154044,
      "perplexity_per_byte_labeled": 16.222603112239046,
      "perplexity_per_byte_ablated": 16.86193446190263
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7394519726435345,
      "eval_nll_ablated": 2.7658262809117633,
      "label_use_gap": 0.02637430826822884,
      "perplexity_per_byte_labeled": 15.47850013004715,
      "perplexity_per_byte_ablated": 15.892165960377513
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8447179635365805,
      "eval_nll_ablated": 2.9091240247090657,
      "label_use_gap": 0.06440606117248526,
      "perplexity_per_byte_labeled": 17.196707886234105,
      "perplexity_per_byte_ablated": 18.340725505882542
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9446823755900065,
      "eval_nll_ablated": 2.968934202194214,
      "label_use_gap": 0.024251826604207327,
      "perplexity_per_byte_labeled": 19.0046250948907,
      "perplexity_per_byte_ablated": 19.47115621764214
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.785761531194051,
      "eval_nll_ablated": 2.8285712718963625,
      "label_use_gap": 0.04280974070231158,
      "perplexity_per_byte_labeled": 16.212159241984626,
      "perplexity_per_byte_ablated": 16.921267655277713
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.734732985496521,
      "eval_nll_ablated": 2.778336000442505,
      "label_use_gap": 0.043603014945984064,
      "perplexity_per_byte_labeled": 15.405629360220422,
      "perplexity_per_byte_ablated": 16.092221207864466
    },
    {
      "train_per_category": 30,
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8385178089141845,
      "eval_nll_ablated": 2.9145955562591555,
      "label_use_gap": 0.07607774734497097,
      "perplexity_per_byte_labeled": 17.09041549348512,
      "perplexity_per_byte_ablated": 18.441352404721062
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9449878692626954,
      "eval_nll_ablated": 2.9449878692626954,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.010431774515794,
      "perplexity_per_byte_ablated": 19.010431774515794
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7495031992594403,
      "eval_nll_ablated": 2.7495031992594403,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.634862543150792,
      "perplexity_per_byte_ablated": 15.634862543150792
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.641265376408895,
      "eval_nll_ablated": 2.641265376408895,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 14.030946808538237,
      "perplexity_per_byte_ablated": 14.030946808538237
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.576992964744568,
      "eval_nll_ablated": 2.576992964744568,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.157513506164511,
      "perplexity_per_byte_ablated": 13.157513506164511
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9541545073191324,
      "eval_nll_ablated": 2.9581135431925456,
      "label_use_gap": 0.003959035873413175,
      "perplexity_per_byte_labeled": 19.18549466513096,
      "perplexity_per_byte_ablated": 19.261601281764438
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7517573595047,
      "eval_nll_ablated": 2.7571523666381834,
      "label_use_gap": 0.00539500713348362,
      "perplexity_per_byte_labeled": 15.6701457808313,
      "perplexity_per_byte_ablated": 15.754914788193473
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.641230495770772,
      "eval_nll_ablated": 2.6490848700205487,
      "label_use_gap": 0.00785437424977653,
      "perplexity_per_byte_labeled": 14.030457408695428,
      "perplexity_per_byte_ablated": 14.141091785214586
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.577167304356893,
      "eval_nll_ablated": 2.5899420976638794,
      "label_use_gap": 0.01277479330698661,
      "perplexity_per_byte_labeled": 13.159807581936665,
      "perplexity_per_byte_ablated": 13.328999800624796
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.954021374384562,
      "eval_nll_ablated": 2.9579098065694174,
      "label_use_gap": 0.0038884321848553505,
      "perplexity_per_byte_labeled": 19.182940613942943,
      "perplexity_per_byte_ablated": 19.257677387897267
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7524760246276854,
      "eval_nll_ablated": 2.7573559125264486,
      "label_use_gap": 0.004879887898763258,
      "perplexity_per_byte_labeled": 15.681411415700644,
      "perplexity_per_byte_ablated": 15.7581219627111
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6423314571380616,
      "eval_nll_ablated": 2.6490036725997923,
      "label_use_gap": 0.006672215461730691,
      "perplexity_per_byte_labeled": 14.045912906659733,
      "perplexity_per_byte_ablated": 14.139943611649944
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5770235935846966,
      "eval_nll_ablated": 2.588712453842163,
      "label_use_gap": 0.011688860257466516,
      "perplexity_per_byte_labeled": 13.157916511713946,
      "perplexity_per_byte_ablated": 13.312619951124564
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9541380246480307,
      "eval_nll_ablated": 2.9580654462178546,
      "label_use_gap": 0.003927421569823952,
      "perplexity_per_byte_labeled": 19.185178439538596,
      "perplexity_per_byte_ablated": 19.26067487929384
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.753271786371867,
      "eval_nll_ablated": 2.7580209811528524,
      "label_use_gap": 0.004749194780985455,
      "perplexity_per_byte_labeled": 15.693895049340313,
      "perplexity_per_byte_ablated": 15.768605681049193
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.642554187774658,
      "eval_nll_ablated": 2.64920388062795,
      "label_use_gap": 0.0066496928532919775,
      "perplexity_per_byte_labeled": 14.04904171021028,
      "perplexity_per_byte_ablated": 14.142774825285088
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5767845551172894,
      "eval_nll_ablated": 2.5866811354955037,
      "label_use_gap": 0.009896580378214281,
      "perplexity_per_byte_labeled": 13.154771639404418,
      "perplexity_per_byte_ablated": 13.2856052290152
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.953963883717855,
      "eval_nll_ablated": 2.9577035109202066,
      "label_use_gap": 0.0037396272023517696,
      "perplexity_per_byte_labeled": 19.18183780559854,
      "perplexity_per_byte_ablated": 19.253705022593167
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.751762366294861,
      "eval_nll_ablated": 2.757045555114746,
      "label_use_gap": 0.0052831888198849875,
      "perplexity_per_byte_labeled": 15.670224238159427,
      "perplexity_per_byte_ablated": 15.753232071611679
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6417994022369387,
      "eval_nll_ablated": 2.648940165837606,
      "label_use_gap": 0.007140763600667288,
      "perplexity_per_byte_labeled": 14.03844169757995,
      "perplexity_per_byte_ablated": 14.139045658127033
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5784090757369995,
      "eval_nll_ablated": 2.588133533795675,
      "label_use_gap": 0.009724458058675278,
      "perplexity_per_byte_labeled": 13.176159204747059,
      "perplexity_per_byte_ablated": 13.304915238984918
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9540479024251303,
      "eval_nll_ablated": 2.9580775419871013,
      "label_use_gap": 0.004029639561970999,
      "perplexity_per_byte_labeled": 19.1834495065197,
      "perplexity_per_byte_ablated": 19.260907853381713
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.752330176035563,
      "eval_nll_ablated": 2.757657353083293,
      "label_use_gap": 0.00532717704772967,
      "perplexity_per_byte_labeled": 15.679124470701083,
      "perplexity_per_byte_ablated": 15.76287281578428
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6419924894968667,
      "eval_nll_ablated": 2.6490485350290935,
      "label_use_gap": 0.007056045532226829,
      "perplexity_per_byte_labeled": 14.041152603533273,
      "perplexity_per_byte_ablated": 14.14057797810005
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.578150200843811,
      "eval_nll_ablated": 2.588529856999715,
      "label_use_gap": 0.010379656155904193,
      "perplexity_per_byte_labeled": 13.172748669410325,
      "perplexity_per_byte_ablated": 13.31018933067534
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9539796829223635,
      "eval_nll_ablated": 2.9578142166137695,
      "label_use_gap": 0.0038345336914060724,
      "perplexity_per_byte_labeled": 19.18214086577093,
      "perplexity_per_byte_ablated": 19.255836635350008
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7527003049850465,
      "eval_nll_ablated": 2.757282026608785,
      "label_use_gap": 0.004581721623738488,
      "perplexity_per_byte_labeled": 15.68492884268703,
      "perplexity_per_byte_ablated": 15.756957702420978
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.64244438012441,
      "eval_nll_ablated": 2.64895339012146,
      "label_use_gap": 0.006509009997050175,
      "perplexity_per_byte_labeled": 14.04749910264845,
      "perplexity_per_byte_ablated": 14.13923263811658
    },
    {
      "train_per_category": 100,
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.577524495124817,
      "eval_nll_ablated": 2.5873711268107096,
      "label_use_gap": 0.009846631685892504,
      "perplexity_per_byte_labeled": 13.16450898331119,
      "perplexity_per_byte_ablated": 13.29477534452669
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.940365759531657,
      "eval_nll_ablated": 2.940365759531657,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 18.922766228778674,
      "perplexity_per_byte_ablated": 18.922766228778674
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7482269366582233,
      "eval_nll_ablated": 2.7482269366582233,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.614921080789909,
      "perplexity_per_byte_ablated": 15.614921080789909
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6387850205103556,
      "eval_nll_ablated": 2.6387850205103556,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.996188191549985,
      "perplexity_per_byte_ablated": 13.996188191549985
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.590206265449524,
      "eval_nll_ablated": 2.590206265449524,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.332521358107526,
      "perplexity_per_byte_ablated": 13.332521358107526
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9446878751118977,
      "eval_nll_ablated": 2.9484476407368976,
      "label_use_gap": 0.003759765624999911,
      "perplexity_per_byte_labeled": 19.004729611529836,
      "perplexity_per_byte_ablated": 19.076317433021426
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.750337529182434,
      "eval_nll_ablated": 2.753761879603068,
      "label_use_gap": 0.003424350420634159,
      "perplexity_per_byte_labeled": 15.647912620090258,
      "perplexity_per_byte_ablated": 15.701588406152235
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6385509252548216,
      "eval_nll_ablated": 2.642715350786845,
      "label_use_gap": 0.00416442553202323,
      "perplexity_per_byte_labeled": 13.992912133768536,
      "perplexity_per_byte_ablated": 14.051306078551946
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5902546485265097,
      "eval_nll_ablated": 2.5948655049006146,
      "label_use_gap": 0.004610856374104966,
      "perplexity_per_byte_labeled": 13.333166442120259,
      "perplexity_per_byte_ablated": 13.394785707218793
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9445831457773846,
      "eval_nll_ablated": 2.9489997227986655,
      "label_use_gap": 0.004416577021280954,
      "perplexity_per_byte_labeled": 19.002739363065537,
      "perplexity_per_byte_ablated": 19.08685203339516
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.751064483324687,
      "eval_nll_ablated": 2.754403026898702,
      "label_use_gap": 0.003338543574014974,
      "perplexity_per_byte_labeled": 15.659292070655273,
      "perplexity_per_byte_ablated": 15.711658665010646
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6387569348017377,
      "eval_nll_ablated": 2.6421348094940185,
      "label_use_gap": 0.003377874692280791,
      "perplexity_per_byte_labeled": 13.99579510420677,
      "perplexity_per_byte_ablated": 14.043151082540206
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.589506157239278,
      "eval_nll_ablated": 2.594212865829468,
      "label_use_gap": 0.004706708590189734,
      "perplexity_per_byte_labeled": 13.323190417156754,
      "perplexity_per_byte_ablated": 13.386046598768628
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9449319362640383,
      "eval_nll_ablated": 2.9491181214650473,
      "label_use_gap": 0.004186185201008996,
      "perplexity_per_byte_labeled": 19.009368493797396,
      "perplexity_per_byte_ablated": 19.0891120250087
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7501022895177205,
      "eval_nll_ablated": 2.753332845369975,
      "label_use_gap": 0.003230555852254291,
      "perplexity_per_byte_labeled": 15.64423204329784,
      "perplexity_per_byte_ablated": 15.69485333210403
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.638565985361735,
      "eval_nll_ablated": 2.6417799949645997,
      "label_use_gap": 0.0032140096028645537,
      "perplexity_per_byte_labeled": 13.993122870108154,
      "perplexity_per_byte_ablated": 14.038169252362428
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.590108291308085,
      "eval_nll_ablated": 2.5951483408610025,
      "label_use_gap": 0.0050400495529174805,
      "perplexity_per_byte_labeled": 13.331215179761147,
      "perplexity_per_byte_ablated": 13.398574770114847
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.945792039235433,
      "eval_nll_ablated": 2.950177796681722,
      "label_use_gap": 0.004385757446288885,
      "perplexity_per_byte_labeled": 19.02572554148753,
      "perplexity_per_byte_ablated": 19.109351005409014
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7503930727640786,
      "eval_nll_ablated": 2.753733587265015,
      "label_use_gap": 0.003340514500936198,
      "perplexity_per_byte_labeled": 15.648781785340493,
      "perplexity_per_byte_ablated": 15.701144177789235
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.639166800181071,
      "eval_nll_ablated": 2.6426302433013915,
      "label_use_gap": 0.0034634431203204308,
      "perplexity_per_byte_labeled": 14.00153267181107,
      "perplexity_per_byte_ablated": 14.050110258111623
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5912283182144167,
      "eval_nll_ablated": 2.5951479355494182,
      "label_use_gap": 0.003919617335001568,
      "perplexity_per_byte_labeled": 13.346154864319143,
      "perplexity_per_byte_ablated": 13.39856933951838
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9453996817270913,
      "eval_nll_ablated": 2.9499698638916017,
      "label_use_gap": 0.004570182164510328,
      "perplexity_per_byte_labeled": 19.018262119480255,
      "perplexity_per_byte_ablated": 19.105377957814824
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7508157889048257,
      "eval_nll_ablated": 2.7543066183725995,
      "label_use_gap": 0.0034908294677737928,
      "perplexity_per_byte_labeled": 15.655398176313291,
      "perplexity_per_byte_ablated": 15.710144000170605
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6398711442947387,
      "eval_nll_ablated": 2.643486738204956,
      "label_use_gap": 0.003615593910217374,
      "perplexity_per_byte_labeled": 14.011398042830935,
      "perplexity_per_byte_ablated": 14.062149260878504
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5921089887619018,
      "eval_nll_ablated": 2.595487856864929,
      "label_use_gap": 0.0033788681030273438,
      "perplexity_per_byte_labeled": 13.357913606859452,
      "perplexity_per_byte_ablated": 13.40312457300085
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9443185806274412,
      "eval_nll_ablated": 2.9486043453216553,
      "label_use_gap": 0.004285764694214045,
      "perplexity_per_byte_labeled": 18.997712565463665,
      "perplexity_per_byte_ablated": 19.079307013657854
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.750615350405375,
      "eval_nll_ablated": 2.753789027531942,
      "label_use_gap": 0.0031736771265666874,
      "perplexity_per_byte_labeled": 15.652260546255965,
      "perplexity_per_byte_ablated": 15.702014677543657
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6402548631032308,
      "eval_nll_ablated": 2.642712203661601,
      "label_use_gap": 0.0024573405583701025,
      "perplexity_per_byte_labeled": 14.016775511445182,
      "perplexity_per_byte_ablated": 14.05126185740146
    },
    {
      "train_per_category": 100,
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5918299277623493,
      "eval_nll_ablated": 2.594825847943624,
      "label_use_gap": 0.0029959201812745917,
      "perplexity_per_byte_labeled": 13.354186454212046,
      "perplexity_per_byte_ablated": 13.39425452131078
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.956036392847697,
      "eval_nll_ablated": 2.956036392847697,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.221633563869432,
      "perplexity_per_byte_ablated": 19.221633563869432
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7676364819208783,
      "eval_nll_ablated": 2.7676364819208783,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.920960028899193,
      "perplexity_per_byte_ablated": 15.920960028899193
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.645840350786845,
      "eval_nll_ablated": 2.645840350786845,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 14.095285091464895,
      "perplexity_per_byte_ablated": 14.095285091464895
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5905239661534627,
      "eval_nll_ablated": 2.5905239661534627,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.33675778245014,
      "perplexity_per_byte_ablated": 13.33675778245014
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9563469886779785,
      "eval_nll_ablated": 2.9592294057210284,
      "label_use_gap": 0.002882417043049923,
      "perplexity_per_byte_labeled": 19.227604650354845,
      "perplexity_per_byte_ablated": 19.28310657711852
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7665007511774697,
      "eval_nll_ablated": 2.7695867379506427,
      "label_use_gap": 0.003085986773172955,
      "perplexity_per_byte_labeled": 15.902888369341994,
      "perplexity_per_byte_ablated": 15.952040274562309
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6456367492675783,
      "eval_nll_ablated": 2.6516229152679442,
      "label_use_gap": 0.0059861660003659445,
      "perplexity_per_byte_labeled": 14.092415562135951,
      "perplexity_per_byte_ablated": 14.17702810071959
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.588236943880717,
      "eval_nll_ablated": 2.6012917200724286,
      "label_use_gap": 0.013054776191711692,
      "perplexity_per_byte_labeled": 13.306291172542736,
      "perplexity_per_byte_ablated": 13.481140652907104
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.956177520751953,
      "eval_nll_ablated": 2.9594698429107664,
      "label_use_gap": 0.0032923221588134766,
      "perplexity_per_byte_labeled": 19.224346464159137,
      "perplexity_per_byte_ablated": 19.287743510496618
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.766826049486796,
      "eval_nll_ablated": 2.7712816476821898,
      "label_use_gap": 0.004455598195393762,
      "perplexity_per_byte_labeled": 15.908062393546983,
      "perplexity_per_byte_ablated": 15.979100468676839
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.646022367477417,
      "eval_nll_ablated": 2.6529977560043334,
      "label_use_gap": 0.006975388526916593,
      "perplexity_per_byte_labeled": 14.097850902113004,
      "perplexity_per_byte_ablated": 14.196532661233292
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5856232484181723,
      "eval_nll_ablated": 2.6062339782714843,
      "label_use_gap": 0.020610729853312026,
      "perplexity_per_byte_labeled": 13.271557990435031,
      "perplexity_per_byte_ablated": 13.54793284701104
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9558959643046063,
      "eval_nll_ablated": 2.9592178503672284,
      "label_use_gap": 0.0033218860626220703,
      "perplexity_per_byte_labeled": 19.218934487390356,
      "perplexity_per_byte_ablated": 19.282883755287052
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7674477895100913,
      "eval_nll_ablated": 2.7714389085769655,
      "label_use_gap": 0.003991119066874216,
      "perplexity_per_byte_labeled": 15.917956147982975,
      "perplexity_per_byte_ablated": 15.98161355391409
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.647159298261007,
      "eval_nll_ablated": 2.65341063340505,
      "label_use_gap": 0.006251335144042969,
      "perplexity_per_byte_labeled": 14.113888297762971,
      "perplexity_per_byte_ablated": 14.20239529892967
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.586704635620117,
      "eval_nll_ablated": 2.603064187367757,
      "label_use_gap": 0.016359551747640033,
      "perplexity_per_byte_labeled": 13.2859174460622,
      "perplexity_per_byte_ablated": 13.505056722782644
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9557055473327636,
      "eval_nll_ablated": 2.9587193648020427,
      "label_use_gap": 0.0030138174692790898,
      "perplexity_per_byte_labeled": 19.21527522448716,
      "perplexity_per_byte_ablated": 19.273273911463097
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.765796589851379,
      "eval_nll_ablated": 2.7686314503351848,
      "label_use_gap": 0.002834860483805546,
      "perplexity_per_byte_labeled": 15.891694112123222,
      "perplexity_per_byte_ablated": 15.936808764441547
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6447768290837605,
      "eval_nll_ablated": 2.6504719654719033,
      "label_use_gap": 0.005695136388142785,
      "perplexity_per_byte_labeled": 14.080302418470122,
      "perplexity_per_byte_ablated": 14.160720439568449
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.586457451184591,
      "eval_nll_ablated": 2.6011221647262572,
      "label_use_gap": 0.01466471354166643,
      "perplexity_per_byte_labeled": 13.282633779910148,
      "perplexity_per_byte_ablated": 13.478855047210725
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9561674276987713,
      "eval_nll_ablated": 2.9593300024668374,
      "label_use_gap": 0.00316257476806614,
      "perplexity_per_byte_labeled": 19.224152432787076,
      "perplexity_per_byte_ablated": 19.285046492462214
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.765554443995158,
      "eval_nll_ablated": 2.7688453515370686,
      "label_use_gap": 0.003290907541910748,
      "perplexity_per_byte_labeled": 15.88784647010972,
      "perplexity_per_byte_ablated": 15.940218031600628
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6452454964319867,
      "eval_nll_ablated": 2.6509440978368124,
      "label_use_gap": 0.005698601404825698,
      "perplexity_per_byte_labeled": 14.086902943071173,
      "perplexity_per_byte_ablated": 14.167407752522616
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.58678719997406,
      "eval_nll_ablated": 2.6006374915440875,
      "label_use_gap": 0.013850291570027373,
      "perplexity_per_byte_labeled": 13.287014434538126,
      "perplexity_per_byte_ablated": 13.472323790533322
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.954704427719116,
      "eval_nll_ablated": 2.958247915903727,
      "label_use_gap": 0.003543488184611032,
      "perplexity_per_byte_labeled": 19.196048061529122,
      "perplexity_per_byte_ablated": 19.264189689252138
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.766357731819153,
      "eval_nll_ablated": 2.7702128251393634,
      "label_use_gap": 0.00385509332021039,
      "perplexity_per_byte_labeled": 15.900614111087384,
      "perplexity_per_byte_ablated": 15.962030769746347
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6459416071573894,
      "eval_nll_ablated": 2.651650826136271,
      "label_use_gap": 0.005709218978881658,
      "perplexity_per_byte_labeled": 14.096712401135921,
      "perplexity_per_byte_ablated": 14.177423799406295
    },
    {
      "train_per_category": 100,
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.585412589708964,
      "eval_nll_ablated": 2.597850783665975,
      "label_use_gap": 0.012438193957010846,
      "perplexity_per_byte_labeled": 13.268762515615482,
      "perplexity_per_byte_ablated": 13.434832622418753
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.956758181254069,
      "eval_nll_ablated": 2.956758181254069,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.23551252436119,
      "perplexity_per_byte_ablated": 19.23551252436119
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.752531369527181,
      "eval_nll_ablated": 2.752531369527181,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.682279325856378,
      "perplexity_per_byte_ablated": 15.682279325856378
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.64053746064504,
      "eval_nll_ablated": 2.64053746064504,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 14.020737177500989,
      "perplexity_per_byte_ablated": 14.020737177500989
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.587854186693827,
      "eval_nll_ablated": 2.587854186693827,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.301199068547465,
      "perplexity_per_byte_ablated": 13.301199068547465
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.959451452891032,
      "eval_nll_ablated": 2.9627869923909507,
      "label_use_gap": 0.003335539499918827,
      "perplexity_per_byte_labeled": 19.287388811774292,
      "perplexity_per_byte_ablated": 19.35183007244462
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7535551230112714,
      "eval_nll_ablated": 2.7576493501663206,
      "label_use_gap": 0.004094227155049257,
      "perplexity_per_byte_labeled": 15.69834233483247,
      "perplexity_per_byte_ablated": 15.762746667326672
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.643657890955607,
      "eval_nll_ablated": 2.650496753056844,
      "label_use_gap": 0.006838862101236742,
      "perplexity_per_byte_labeled": 14.064556242379448,
      "perplexity_per_byte_ablated": 14.161071453979547
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.590233079592387,
      "eval_nll_ablated": 2.6112543106079102,
      "label_use_gap": 0.021021231015523334,
      "perplexity_per_byte_labeled": 13.332878863033018,
      "perplexity_per_byte_ablated": 13.616118987706063
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9586425463358563,
      "eval_nll_ablated": 2.962076250712077,
      "label_use_gap": 0.003433704376220703,
      "perplexity_per_byte_labeled": 19.27179342498791,
      "perplexity_per_byte_ablated": 19.33808080691648
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.752688749631246,
      "eval_nll_ablated": 2.757725238800049,
      "label_use_gap": 0.005036489168802838,
      "perplexity_per_byte_labeled": 15.684747598832088,
      "perplexity_per_byte_ablated": 15.763942926025704
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6430310010910034,
      "eval_nll_ablated": 2.6488662004470824,
      "label_use_gap": 0.005835199356079013,
      "perplexity_per_byte_labeled": 14.055742077664869,
      "perplexity_per_byte_ablated": 14.13799989676885
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5908344745635987,
      "eval_nll_ablated": 2.604034725824992,
      "label_use_gap": 0.013200251261393348,
      "perplexity_per_byte_labeled": 13.340899600906992,
      "perplexity_per_byte_ablated": 13.51817026226708
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9588947455088297,
      "eval_nll_ablated": 2.962575912475586,
      "label_use_gap": 0.003681166966756244,
      "perplexity_per_byte_labeled": 19.27665436828858,
      "perplexity_per_byte_ablated": 19.347745720868236
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.753426957130432,
      "eval_nll_ablated": 2.75741130510966,
      "label_use_gap": 0.003984347979228087,
      "perplexity_per_byte_labeled": 15.696330471888254,
      "perplexity_per_byte_ablated": 15.758994869969245
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.643447987238566,
      "eval_nll_ablated": 2.6492788791656494,
      "label_use_gap": 0.005830891927083304,
      "perplexity_per_byte_labeled": 14.061604349563114,
      "perplexity_per_byte_ablated": 14.143835552491993
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5913436810175576,
      "eval_nll_ablated": 2.6030776500701904,
      "label_use_gap": 0.011733969052632798,
      "perplexity_per_byte_labeled": 13.347694602968021,
      "perplexity_per_byte_ablated": 13.50523853856651
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9596819559733074,
      "eval_nll_ablated": 2.9631860733032225,
      "label_use_gap": 0.0035041173299150685,
      "perplexity_per_byte_labeled": 19.291835126769424,
      "perplexity_per_byte_ablated": 19.35955455968925
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7537558555603026,
      "eval_nll_ablated": 2.7577246983846027,
      "label_use_gap": 0.003968842824300101,
      "perplexity_per_byte_labeled": 15.701493819397088,
      "perplexity_per_byte_ablated": 15.763934406949756
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.643793153762817,
      "eval_nll_ablated": 2.6488032420476277,
      "label_use_gap": 0.005010088284810532,
      "perplexity_per_byte_labeled": 14.066458782407505,
      "perplexity_per_byte_ablated": 14.13710981894309
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.591361125310262,
      "eval_nll_ablated": 2.604673894246419,
      "label_use_gap": 0.01331276893615696,
      "perplexity_per_byte_labeled": 13.347927446090491,
      "perplexity_per_byte_ablated": 13.526813411734043
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9593294779459636,
      "eval_nll_ablated": 2.9627151330312094,
      "label_use_gap": 0.003385655085245798,
      "perplexity_per_byte_labeled": 19.28503637705543,
      "perplexity_per_byte_ablated": 19.350439512288773
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.753555146853129,
      "eval_nll_ablated": 2.7575534184773765,
      "label_use_gap": 0.003998271624247263,
      "perplexity_per_byte_labeled": 15.69834270911012,
      "perplexity_per_byte_ablated": 15.76123459294556
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.644384026527405,
      "eval_nll_ablated": 2.649117382367452,
      "label_use_gap": 0.004733355840047082,
      "perplexity_per_byte_labeled": 14.074772725795693,
      "perplexity_per_byte_ablated": 14.141551552770318
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5931126991907756,
      "eval_nll_ablated": 2.6029723564783733,
      "label_use_gap": 0.009859657287597745,
      "perplexity_per_byte_labeled": 13.371327814918752,
      "perplexity_per_byte_ablated": 13.503816598354348
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.958650588989258,
      "eval_nll_ablated": 2.962300173441569,
      "label_use_gap": 0.0036495844523112275,
      "perplexity_per_byte_labeled": 19.271948421966144,
      "perplexity_per_byte_ablated": 19.342411527609205
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.752752923965454,
      "eval_nll_ablated": 2.7567670742670694,
      "label_use_gap": 0.004014150301615338,
      "perplexity_per_byte_labeled": 15.685754189364758,
      "perplexity_per_byte_ablated": 15.74884570897807
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6443412621816,
      "eval_nll_ablated": 2.6504115502039594,
      "label_use_gap": 0.006070288022359538,
      "perplexity_per_byte_labeled": 14.074170840217434,
      "perplexity_per_byte_ablated": 14.15986494169164
    },
    {
      "train_per_category": 100,
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5927035172780353,
      "eval_nll_ablated": 2.6060229698816935,
      "label_use_gap": 0.013319452603658188,
      "perplexity_per_byte_labeled": 13.365857628654528,
      "perplexity_per_byte_ablated": 13.545074421102528
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.951421022415161,
      "eval_nll_ablated": 2.951421022415161,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.133123016254217,
      "perplexity_per_byte_ablated": 19.133123016254217
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.751519274711609,
      "eval_nll_ablated": 2.751519274711609,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.666415401506285,
      "perplexity_per_byte_ablated": 15.666415401506285
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6481795152028402,
      "eval_nll_ablated": 2.6481795152028402,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 14.128294873386173,
      "perplexity_per_byte_ablated": 14.128294873386173
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5882457812627155,
      "eval_nll_ablated": 2.5882457812627155,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.30640876584042,
      "perplexity_per_byte_ablated": 13.30640876584042
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9598045190175375,
      "eval_nll_ablated": 2.982477617263794,
      "label_use_gap": 0.02267309824625663,
      "perplexity_per_byte_labeled": 19.294199737715342,
      "perplexity_per_byte_ablated": 19.736655996795033
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.756444787979126,
      "eval_nll_ablated": 2.8123215277989706,
      "label_use_gap": 0.055876739819844534,
      "perplexity_per_byte_labeled": 15.743770889771382,
      "perplexity_per_byte_ablated": 16.64852339983221
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6478674252827963,
      "eval_nll_ablated": 2.7331281661987306,
      "label_use_gap": 0.08526074091593427,
      "perplexity_per_byte_labeled": 14.123886262946003,
      "perplexity_per_byte_ablated": 15.380925936495949
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5873547395070395,
      "eval_nll_ablated": 2.661402726173401,
      "label_use_gap": 0.07404798666636125,
      "perplexity_per_byte_labeled": 13.294557480790997,
      "perplexity_per_byte_ablated": 14.316356949137836
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9585561593373617,
      "eval_nll_ablated": 2.9819234053293866,
      "label_use_gap": 0.023367245992024888,
      "perplexity_per_byte_labeled": 19.27012866450618,
      "perplexity_per_byte_ablated": 19.725720737001954
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.757575496037801,
      "eval_nll_ablated": 2.812473980585734,
      "label_use_gap": 0.054898484547933,
      "perplexity_per_byte_labeled": 15.761582566395841,
      "perplexity_per_byte_ablated": 16.651061707101103
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.648537445068359,
      "eval_nll_ablated": 2.7328911145528156,
      "label_use_gap": 0.08435366948445644,
      "perplexity_per_byte_labeled": 14.133352717192896,
      "perplexity_per_byte_ablated": 15.377280294806761
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5878161509831745,
      "eval_nll_ablated": 2.6509667158126833,
      "label_use_gap": 0.06315056482950876,
      "perplexity_per_byte_labeled": 13.300693157609762,
      "perplexity_per_byte_ablated": 14.167728194233174
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9592453638712564,
      "eval_nll_ablated": 2.9820934931437173,
      "label_use_gap": 0.022848129272460938,
      "perplexity_per_byte_labeled": 19.283414302285497,
      "perplexity_per_byte_ablated": 19.7290761270756
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7580661217371625,
      "eval_nll_ablated": 2.8130550622940063,
      "label_use_gap": 0.05498894055684378,
      "perplexity_per_byte_labeled": 15.769317501189258,
      "perplexity_per_byte_ablated": 16.66074014619204
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6485588630040486,
      "eval_nll_ablated": 2.732895533243815,
      "label_use_gap": 0.08433667023976632,
      "perplexity_per_byte_labeled": 14.133655427674174,
      "perplexity_per_byte_ablated": 15.377348242406912
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5868266423543296,
      "eval_nll_ablated": 2.655639425913493,
      "label_use_gap": 0.0688127835591632,
      "perplexity_per_byte_labeled": 13.287538516349546,
      "perplexity_per_byte_ablated": 14.234084792708865
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9600055694580076,
      "eval_nll_ablated": 2.9831733862559,
      "label_use_gap": 0.02316781679789237,
      "perplexity_per_byte_labeled": 19.298079235045385,
      "perplexity_per_byte_ablated": 19.750392928356813
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7567930539449055,
      "eval_nll_ablated": 2.8104497671127318,
      "label_use_gap": 0.053656713167826275,
      "perplexity_per_byte_labeled": 15.74925486423072,
      "perplexity_per_byte_ablated": 16.617390494012536
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6489962657292683,
      "eval_nll_ablated": 2.728320574760437,
      "label_use_gap": 0.07932430903116883,
      "perplexity_per_byte_labeled": 14.139838879306266,
      "perplexity_per_byte_ablated": 15.307158193318257
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5868301312128703,
      "eval_nll_ablated": 2.6464789231618244,
      "label_use_gap": 0.059648791948954116,
      "perplexity_per_byte_labeled": 13.287584874772653,
      "perplexity_per_byte_ablated": 14.10428882560375
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.959835418065389,
      "eval_nll_ablated": 2.982262452443441,
      "label_use_gap": 0.022427034378051935,
      "perplexity_per_byte_labeled": 19.29479591932697,
      "perplexity_per_byte_ablated": 19.73240981958347
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7574071645736695,
      "eval_nll_ablated": 2.807522741953532,
      "label_use_gap": 0.0501155773798625,
      "perplexity_per_byte_labeled": 15.758929619418897,
      "perplexity_per_byte_ablated": 16.568822089103588
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.649130622545878,
      "eval_nll_ablated": 2.7229182004928587,
      "label_use_gap": 0.07378757794698076,
      "perplexity_per_byte_labeled": 14.141738790675626,
      "perplexity_per_byte_ablated": 15.224686168739801
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5860597769419353,
      "eval_nll_ablated": 2.6401027838389077,
      "label_use_gap": 0.05404300689697239,
      "perplexity_per_byte_labeled": 13.277352668731796,
      "perplexity_per_byte_ablated": 14.014644012619716
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.959878174463908,
      "eval_nll_ablated": 2.981920846303304,
      "label_use_gap": 0.02204267183939601,
      "perplexity_per_byte_labeled": 19.295620912947392,
      "perplexity_per_byte_ablated": 19.72567025843268
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7575608174006145,
      "eval_nll_ablated": 2.8086196899414064,
      "label_use_gap": 0.05105887254079189,
      "perplexity_per_byte_labeled": 15.761351209541868,
      "perplexity_per_byte_ablated": 16.58700719739115
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6487020174662272,
      "eval_nll_ablated": 2.728815960884094,
      "label_use_gap": 0.08011394341786682,
      "perplexity_per_byte_labeled": 14.135678868344366,
      "perplexity_per_byte_ablated": 15.314743025635073
    },
    {
      "train_per_category": 100,
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5859323263168337,
      "eval_nll_ablated": 2.649219266573588,
      "label_use_gap": 0.06328694025675441,
      "perplexity_per_byte_labeled": 13.275660569666302,
      "perplexity_per_byte_ablated": 14.142992426923719
    }
  ]
}