{
  "experiment": "H1 real-text conditional byte-model pilot v12 (v11 structure, different category groups)",
  "completed_utc": "2026-11-19T00:43:53.204704+00:00",
  "smoke_test_not_protocol": false,
  "protocol": {
    "dataset": "SetFit/20_newsgroups",
    "revision": "f1b91292074e7cfb69be58b642d583ec262f30ed",
    "dataset_split": "train only; official test unused",
    "selected_data_sha256": "9cbd6e8cf8d80bad34613d7930aa54da65872f8719dc62fb92bebea8f8071858",
    "categories": [
      "sci.crypt",
      "sci.electronics",
      "sci.med",
      "sci.space",
      "talk.politics.guns",
      "talk.politics.mideast",
      "talk.politics.misc",
      "soc.religion.christian"
    ],
    "sizes_train_per_category": [
      30,
      100
    ],
    "subsets_nested": true,
    "validation_per_category": 80,
    "validation_identical_across_sizes": true,
    "validation_messages": 640,
    "bytes_per_example": 96,
    "seed_values": [
      12101,
      13103,
      14107,
      15109,
      16111
    ],
    "steps": [
      100,
      200,
      400,
      800
    ],
    "total_updates": 800,
    "batch_size": 24,
    "conditions": [
      "none",
      "hierarchy",
      "scrambled_1",
      "scrambled_2",
      "scrambled_3",
      "scrambled_4",
      "scrambled_5"
    ],
    "permutation_seeds_selected": [
      10101,
      20202,
      30303,
      40404,
      50505
    ],
    "permutation_seeds_rejected": [],
    "permutation_seeds_primary": [
      10101,
      20202,
      30303,
      40404,
      50505
    ],
    "permutation_seeds_fallback": [
      60606,
      70707,
      80808,
      90909
    ],
    "scrambled_permutations": {
      "scrambled_1": [
        6,
        0,
        1,
        7,
        5,
        2,
        4,
        3
      ],
      "scrambled_2": [
        7,
        3,
        1,
        6,
        0,
        5,
        2,
        4
      ],
      "scrambled_3": [
        7,
        4,
        1,
        6,
        2,
        3,
        5,
        0
      ],
      "scrambled_4": [
        2,
        0,
        1,
        5,
        3,
        4,
        6,
        7
      ],
      "scrambled_5": [
        3,
        2,
        5,
        6,
        1,
        4,
        7,
        0
      ]
    },
    "permutation_rejection_rule": "rejected if categories 0-3 all fall in the same code//4 group (group split preserved)",
    "same_batch_sequence_per_size_and_seed_across_conditions": true,
    "batch_generator_seed": "seed + 30000, per (size, seed)",
    "evaluations": "each checkpoint: (a) eval_nll_labeled with the condition label prefix; (b) eval_nll_ablated with dummy prefix [256, 256] for all conditions; label_use_gap = ablated - labeled",
    "metric": "validation NLL in nats per byte, lower is better",
    "model": "randomly initialized 2-layer transformer width 64, 4 heads, FF 128, dropout 0.2 in training only (eval mode at every evaluation)",
    "dropout": 0.2,
    "weight_decay": 0.1,
    "learning_rate": 0.001,
    "regularisation_chosen_before_run": true,
    "v9_reference": "v9 used dropout 0 and weight decay 0.01",
    "replication_of": "v11 structure, different category groups",
    "category_groups": {
      "group_0": [
        "sci.crypt",
        "sci.electronics",
        "sci.med",
        "sci.space"
      ],
      "group_1": [
        "talk.politics.guns",
        "talk.politics.mideast",
        "talk.politics.misc",
        "soc.religion.christian"
      ]
    },
    "category_set_differs_from_v10_v11": true,
    "preregistered_cells": [
      {
        "train_per_category": 30,
        "step": 200
      },
      {
        "train_per_category": 30,
        "step": 400
      },
      {
        "train_per_category": 100,
        "step": 200
      },
      {
        "train_per_category": 100,
        "step": 400
      }
    ],
    "new_seeds_and_permutations_vs_v10": true,
    "new_seeds_and_permutations_vs_v11": true,
    "v10_seeds_not_reused": [
      2027,
      3037,
      4049,
      5051,
      6067
    ],
    "v11_seeds_not_reused": [
      7079,
      8081,
      9091,
      10093,
      11099
    ],
    "primary_checkpoint": 800,
    "checkpoint_selection": "none; all checkpoints are reported and the primary criterion is fixed in advance at 800 updates",
    "limitations": "a single regularisation setting (dropout 0.2, weight decay 0.1), unchanged from v10 and not tuned; the new grouping (sci vs politics/society) may be more or less semantically clean than comp vs rec; five seeds, five permutations, two groups of four categories, short prefixes, a tiny model; one historical text dataset (20 Newsgroups); no significance test is run or claimed; no data-savings percentage is estimated; labels present at labeled evaluation; the four cells come from v10/v11 and were chosen AFTER seeing v10; comparison with v10/v11 is unpaired because the categories change; the cells share seeds across the two checkpoints and are not independent; a null result is informative; not a pretrained-LLM or general data-savings result"
  },
  "environment": {
    "device": "cuda",
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition MIG 2g.48gb",
    "torch": "2.13.0+cu130",
    "cuda": "13.0",
    "python": "3.12.12",
    "code_sha256": "178853b12669ea2335b71100d9c98b7ae7abf6c791d3537e6fc3f4942219070f"
  },
  "summary": {
    "30": {
      "100": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.964281924565633,
          "hierarchy": 2.970807196299235,
          "scrambled_1": 2.9710207112630207,
          "scrambled_2": 2.970810260772705,
          "scrambled_3": 2.9712304941813152,
          "scrambled_4": 2.970885025660197,
          "scrambled_5": 2.970589946111043,
          "scrambled_mean": 2.9709072875976563
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.00010009129842138975,
          "min": -0.0009212176005042316,
          "max": 0.0004702536265051016,
          "per_seed": {
            "12101": -3.124872843462967e-05,
            "13103": 0.0004702536265051016,
            "14107": -5.6896209716938984e-05,
            "15109": -0.0009212176005042316,
            "16111": 3.865242004374991e-05
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 2.0,
          "per_seed": {
            "12101": 1,
            "13103": 4,
            "14107": 2,
            "15109": 0,
            "16111": 3
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 3.0,
          "per_seed": {
            "12101": 2,
            "13103": 5,
            "14107": 3,
            "15109": 1,
            "16111": 4
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0010622660319010713,
          "per_seed": {
            "12101": 0.0009607156117756865,
            "13103": 0.0016903400421144354,
            "14107": 0.0008227348327638495,
            "15109": 0.000331878662109375,
            "16111": 0.0015056610107420099
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 3
      },
      "200": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.780846529006958,
          "hierarchy": 2.784168308575948,
          "scrambled_1": 2.7847198915481566,
          "scrambled_2": 2.7847948694229125,
          "scrambled_3": 2.785273405710856,
          "scrambled_4": 2.783557472229004,
          "scrambled_5": 2.7840665197372436,
          "scrambled_mean": 2.784482431729635
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.00031412315368655185,
          "min": -0.0013633171717328807,
          "max": 0.0012319262822471089,
          "per_seed": {
            "12101": 0.0012319262822471089,
            "13103": -0.0005734316507974846,
            "14107": -0.0013633171717328807,
            "15109": -8.725325266523143e-05,
            "16111": -0.0007785399754842715
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.6,
          "per_seed": {
            "12101": 5,
            "13103": 1,
            "14107": 1,
            "15109": 1,
            "16111": 0
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.6,
          "per_seed": {
            "12101": 6,
            "13103": 2,
            "14107": 2,
            "15109": 2,
            "16111": 1
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0019355154037476118,
          "per_seed": {
            "12101": 0.0018863280614218603,
            "13103": 0.0014224847157797704,
            "14107": 0.0030641396840409563,
            "15109": 0.0020084222157796816,
            "16111": 0.0012962023417157909
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 4
      },
      "400": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.729363004366557,
          "hierarchy": 2.7377207692464194,
          "scrambled_1": 2.739714447657267,
          "scrambled_2": 2.7390962489446005,
          "scrambled_3": 2.7407992267608643,
          "scrambled_4": 2.7375719022750853,
          "scrambled_5": 2.7396320358912147,
          "scrambled_mean": 2.7393627723058063
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0016420030593872425,
          "min": -0.0023271099726356326,
          "max": -0.0010938119888304065,
          "per_seed": {
            "12101": -0.0023271099726356326,
            "13103": -0.00117328643798853,
            "14107": -0.0010938119888304065,
            "15109": -0.001643975575765122,
            "16111": -0.0019718313217165218
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.0,
          "per_seed": {
            "12101": 0,
            "13103": 1,
            "14107": 1,
            "15109": 2,
            "16111": 1
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.0,
          "per_seed": {
            "12101": 1,
            "13103": 2,
            "14107": 2,
            "15109": 3,
            "16111": 2
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.004420177141825477,
          "per_seed": {
            "12101": 0.004749544461568167,
            "13103": 0.0034497419993084755,
            "14107": 0.0039218505223592715,
            "15109": 0.005488332112630445,
            "16111": 0.004491416613261023
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 5
      },
      "800": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.8314466667175293,
          "hierarchy": 2.8489892927805585,
          "scrambled_1": 2.850883566538493,
          "scrambled_2": 2.8520209852854412,
          "scrambled_3": 2.8540895716349284,
          "scrambled_4": 2.8505388832092287,
          "scrambled_5": 2.8517437489827473,
          "scrambled_mean": 2.8518553511301676
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0028660583496092685,
          "min": -0.011065883636474538,
          "max": 0.0024067910512290958,
          "per_seed": {
            "12101": 0.0023762861887615117,
            "13103": -0.0024987538655598662,
            "14107": -0.005548731486002545,
            "15109": 0.0024067910512290958,
            "16111": -0.011065883636474538
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 2.0,
          "per_seed": {
            "12101": 5,
            "13103": 1,
            "14107": 0,
            "15109": 4,
            "16111": 0
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 3.0,
          "per_seed": {
            "12101": 6,
            "13103": 2,
            "14107": 1,
            "15109": 5,
            "16111": 1
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.007402782440185618,
          "per_seed": {
            "12101": 0.002736647923787583,
            "13103": 0.01195281346639021,
            "14107": 0.006941461563110263,
            "15109": 0.00699914296468096,
            "16111": 0.008383846282959073
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 3
      }
    },
    "100": {
      "100": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.9463080565134683,
          "hierarchy": 2.952481225331624,
          "scrambled_1": 2.9527835496266683,
          "scrambled_2": 2.9528719011942544,
          "scrambled_3": 2.9532298723856605,
          "scrambled_4": 2.952887398401896,
          "scrambled_5": 2.953228232065837,
          "scrambled_mean": 2.9530001907348633
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.000518965403238969,
          "min": -0.002263476053873781,
          "max": 3.358523050911444e-05,
          "per_seed": {
            "12101": -0.002263476053873781,
            "13103": -3.4354527790991796e-05,
            "14107": -1.5230178832759123e-05,
            "15109": -0.0003153514862064277,
            "16111": 3.358523050911444e-05
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.8,
          "per_seed": {
            "12101": 0,
            "13103": 3,
            "14107": 2,
            "15109": 1,
            "16111": 3
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.8,
          "per_seed": {
            "12101": 1,
            "13103": 4,
            "14107": 3,
            "15109": 2,
            "16111": 4
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.001197217305501308,
          "per_seed": {
            "12101": 0.0023190498352048117,
            "13103": 0.0019713242848715673,
            "14107": 0.0005338668823240411,
            "15109": 0.0009115695953370029,
            "16111": 0.00025027592976911706
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 4
      },
      "200": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.7474950234095257,
          "hierarchy": 2.7478867546717325,
          "scrambled_1": 2.7476443910598753,
          "scrambled_2": 2.7481130250295003,
          "scrambled_3": 2.7481907924016316,
          "scrambled_4": 2.7480747858683268,
          "scrambled_5": 2.7482066249847414,
          "scrambled_mean": 2.748045923868815
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0001591691970824982,
          "min": -0.0007768567403156901,
          "max": 0.0002104854583739879,
          "per_seed": {
            "12101": -0.00011337280273426842,
            "13103": -0.00011596361796062382,
            "14107": 0.0002104854583739879,
            "15109": -0.0007768567403156901,
            "16111": -1.3828277589666982e-07
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 2.2,
          "per_seed": {
            "12101": 2,
            "13103": 2,
            "14107": 4,
            "15109": 1,
            "16111": 2
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 3.2,
          "per_seed": {
            "12101": 3,
            "13103": 3,
            "14107": 5,
            "15109": 2,
            "16111": 3
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0009725268681843424,
          "per_seed": {
            "12101": 0.0005604028701782227,
            "13103": 0.0005481640497841944,
            "14107": 0.0009291728337603722,
            "15109": 0.0018049875895185252,
            "16111": 0.0010199069976803976
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 4
      },
      "400": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.6340543444951376,
          "hierarchy": 2.634595979054769,
          "scrambled_1": 2.634906841913859,
          "scrambled_2": 2.634664543469747,
          "scrambled_3": 2.634820008277893,
          "scrambled_4": 2.6347818771998086,
          "scrambled_5": 2.634704648653666,
          "scrambled_mean": 2.6347755839029947
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0001796048482258783,
          "min": -0.00040332953135147065,
          "max": 0.0002638355890907107,
          "per_seed": {
            "12101": 0.0002638355890907107,
            "13103": -0.00040332953135147065,
            "14107": -0.00016694704691522233,
            "15109": -0.00029335498809812677,
            "16111": -0.00029822826385528245
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 2.0,
          "per_seed": {
            "12101": 5,
            "13103": 1,
            "14107": 2,
            "15109": 1,
            "16111": 1
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 3.0,
          "per_seed": {
            "12101": 6,
            "13103": 2,
            "14107": 3,
            "15109": 2,
            "16111": 2
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.000807889302571585,
          "per_seed": {
            "12101": 0.0002249956130979669,
            "13103": 0.0007021427154541016,
            "14107": 0.0010842402776081173,
            "15109": 0.0012794653574625947,
            "16111": 0.0007486025492351445
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 4
      },
      "800": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.567911173502604,
          "hierarchy": 2.568283019065857,
          "scrambled_1": 2.5688564364115396,
          "scrambled_2": 2.56797204653422,
          "scrambled_3": 2.5683123207092287,
          "scrambled_4": 2.5690154107411702,
          "scrambled_5": 2.5681218687693277,
          "scrambled_mean": 2.5684556166330976
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.00017259756724046226,
          "min": -0.0014019076029461175,
          "max": 0.0008854118982952741,
          "per_seed": {
            "12101": 0.0003108819325761125,
            "13103": -0.0007087628046669181,
            "14107": -0.0014019076029461175,
            "15109": 5.138874053933762e-05,
            "16111": 0.0008854118982952741
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 2.2,
          "per_seed": {
            "12101": 3,
            "13103": 0,
            "14107": 0,
            "15109": 3,
            "16111": 5
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 3.2,
          "per_seed": {
            "12101": 4,
            "13103": 1,
            "14107": 1,
            "15109": 4,
            "16111": 6
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.002054355939229424,
          "per_seed": {
            "12101": 0.005502200126648038,
            "13103": 0.0012825330098471532,
            "14107": 0.0012057224909463038,
            "15109": 0.0016402641932171669,
            "16111": 0.0006410598754884589
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 2
      }
    }
  },
  "replication_cells": {
    "cells": {
      "30_200": {
        "train_per_category": 30,
        "step": 200,
        "mean_paired_diff": -0.00031412315368655185,
        "seeds_hierarchy_better": 4,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.0019355154037476118,
        "sign_negative": true,
        "wins_at_least_4_of_5": true,
        "material": false
      },
      "30_400": {
        "train_per_category": 30,
        "step": 400,
        "mean_paired_diff": -0.0016420030593872425,
        "seeds_hierarchy_better": 5,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.004420177141825477,
        "sign_negative": true,
        "wins_at_least_4_of_5": true,
        "material": false
      },
      "100_200": {
        "train_per_category": 100,
        "step": 200,
        "mean_paired_diff": -0.0001591691970824982,
        "seeds_hierarchy_better": 4,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.0009725268681843424,
        "sign_negative": true,
        "wins_at_least_4_of_5": true,
        "material": false
      },
      "100_400": {
        "train_per_category": 100,
        "step": 400,
        "mean_paired_diff": -0.0001796048482258783,
        "seeds_hierarchy_better": 4,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.000807889302571585,
        "sign_negative": true,
        "wins_at_least_4_of_5": true,
        "material": false
      }
    },
    "missing_cells": [],
    "n_cells_sign_negative": 4,
    "n_cells_wins_at_least_4_of_5": 4,
    "n_cells_material": 0
  },
  "summary_label_use": {
    "30": {
      "100": {
        "none": 0.0,
        "hierarchy": 0.0047225062052409505,
        "scrambled_1": 0.004825805028279451,
        "scrambled_2": 0.004785928726196343,
        "scrambled_3": 0.004458424250284842,
        "scrambled_4": 0.00468974431355802,
        "scrambled_5": 0.004593753814697265,
        "scrambled_mean": 0.004670731226603184
      },
      "200": {
        "none": 0.0,
        "hierarchy": 0.011246204376220615,
        "scrambled_1": 0.010053173700968542,
        "scrambled_2": 0.007355524698893223,
        "scrambled_3": 0.008253728548685757,
        "scrambled_4": 0.00957016944885245,
        "scrambled_5": 0.008638442357381227,
        "scrambled_mean": 0.00877420775095624
      },
      "400": {
        "none": 0.0,
        "hierarchy": 0.01716426849365238,
        "scrambled_1": 0.013254092534383144,
        "scrambled_2": 0.011000326474507638,
        "scrambled_3": 0.01300021648406977,
        "scrambled_4": 0.014720004399617449,
        "scrambled_5": 0.010441884994506889,
        "scrambled_mean": 0.012483304977416978
      },
      "800": {
        "none": 0.0,
        "hierarchy": 0.018667306900024362,
        "scrambled_1": 0.021707515716552807,
        "scrambled_2": 0.02212754567464188,
        "scrambled_3": 0.020428336461385133,
        "scrambled_4": 0.019966878890991246,
        "scrambled_5": 0.02302692731221532,
        "scrambled_mean": 0.021451440811157275
      }
    },
    "100": {
      "100": {
        "none": 0.0,
        "hierarchy": 0.004602231979370064,
        "scrambled_1": 0.004712057113647283,
        "scrambled_2": 0.004789085388183611,
        "scrambled_3": 0.004722983042399242,
        "scrambled_4": 0.004860146840413293,
        "scrambled_5": 0.004629163742065412,
        "scrambled_mean": 0.0047426872253417685
      },
      "200": {
        "none": 0.0,
        "hierarchy": 0.00832181612650551,
        "scrambled_1": 0.008720135688781827,
        "scrambled_2": 0.007710908253987636,
        "scrambled_3": 0.008297390937805104,
        "scrambled_4": 0.008056124051411918,
        "scrambled_5": 0.008648659388224279,
        "scrambled_mean": 0.008286643664042153
      },
      "400": {
        "none": 0.0,
        "hierarchy": 0.00942198912302672,
        "scrambled_1": 0.010967728296915702,
        "scrambled_2": 0.009003701210022096,
        "scrambled_3": 0.009530852635701503,
        "scrambled_4": 0.009772802988688234,
        "scrambled_5": 0.009867909749349035,
        "scrambled_mean": 0.009828598976135314
      },
      "800": {
        "none": 0.0,
        "hierarchy": 0.010347879727681608,
        "scrambled_1": 0.011142570177713917,
        "scrambled_2": 0.010282535552978445,
        "scrambled_3": 0.011970745722452936,
        "scrambled_4": 0.010823828379313305,
        "scrambled_5": 0.010982410113016794,
        "scrambled_mean": 0.011040417989095079
      }
    }
  },
  "elapsed_seconds": 248.59300790300767,
  "runs": [
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.0032374699910482,
      "eval_nll_ablated": 3.0032374699910482,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 20.15066862029976,
      "perplexity_per_byte_ablated": 20.15066862029976
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7693985144297284,
      "eval_nll_ablated": 2.7693985144297284,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.949038007933526,
      "perplexity_per_byte_ablated": 15.949038007933526
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7248002370198567,
      "eval_nll_ablated": 2.7248002370198567,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.253366564532053,
      "perplexity_per_byte_ablated": 15.253366564532053
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8312987168629964,
      "eval_nll_ablated": 2.8312987168629964,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.967482477115603,
      "perplexity_per_byte_ablated": 16.967482477115603
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.020403782526652,
      "eval_nll_ablated": 3.022378142674764,
      "label_use_gap": 0.00197436014811192,
      "perplexity_per_byte_labeled": 20.499567380401867,
      "perplexity_per_byte_ablated": 20.540080890260462
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7756590366363527,
      "eval_nll_ablated": 2.781676514943441,
      "label_use_gap": 0.006017478307088098,
      "perplexity_per_byte_labeled": 16.049200522230596,
      "perplexity_per_byte_ablated": 16.146067393070233
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7323219537734986,
      "eval_nll_ablated": 2.7412479639053347,
      "label_use_gap": 0.008926010131836026,
      "perplexity_per_byte_labeled": 15.368530640186712,
      "perplexity_per_byte_ablated": 15.506324359583928
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8543517271677654,
      "eval_nll_ablated": 2.873370806376139,
      "label_use_gap": 0.019019079208373757,
      "perplexity_per_byte_labeled": 17.363177482120584,
      "perplexity_per_byte_ablated": 17.696569484780113
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.020556163787842,
      "eval_nll_ablated": 3.022823365529378,
      "label_use_gap": 0.0022672017415361623,
      "perplexity_per_byte_labeled": 20.50269136834571,
      "perplexity_per_byte_ablated": 20.549227839772826
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7748045762379965,
      "eval_nll_ablated": 2.7781543175379437,
      "label_use_gap": 0.003349741299947251,
      "perplexity_per_byte_labeled": 16.03549297307202,
      "perplexity_per_byte_ablated": 16.089297791949967
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7347336212793985,
      "eval_nll_ablated": 2.7394805431365965,
      "label_use_gap": 0.00474692185719805,
      "perplexity_per_byte_labeled": 15.405639154858898,
      "perplexity_per_byte_ablated": 15.478942364745128
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8507012049357097,
      "eval_nll_ablated": 2.866403563817342,
      "label_use_gap": 0.015702358881632517,
      "perplexity_per_byte_labeled": 17.299908369615856,
      "perplexity_per_byte_ablated": 17.573701714525065
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.0207200845082602,
      "eval_nll_ablated": 3.02268492380778,
      "label_use_gap": 0.0019648392995197383,
      "perplexity_per_byte_labeled": 20.50605245975406,
      "perplexity_per_byte_ablated": 20.54638316620847
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7747596581776937,
      "eval_nll_ablated": 2.7785382668177285,
      "label_use_gap": 0.003778608640034875,
      "perplexity_per_byte_labeled": 16.034772706008294,
      "perplexity_per_byte_ablated": 16.095476452319627
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7330999930699664,
      "eval_nll_ablated": 2.7397958358128864,
      "label_use_gap": 0.006695842742920011,
      "perplexity_per_byte_labeled": 15.380492613793086,
      "perplexity_per_byte_ablated": 15.483823531367054
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.852273464202881,
      "eval_nll_ablated": 2.8754603703816732,
      "label_use_gap": 0.023186906178792377,
      "perplexity_per_byte_labeled": 17.327129704761827,
      "perplexity_per_byte_ablated": 17.733586260389107
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.0206852753957114,
      "eval_nll_ablated": 3.022370481491089,
      "label_use_gap": 0.0016852060953773673,
      "perplexity_per_byte_labeled": 20.50533867468924,
      "perplexity_per_byte_ablated": 20.53992352953085
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.775215498606364,
      "eval_nll_ablated": 2.7785733540852866,
      "label_use_gap": 0.0033578554789226445,
      "perplexity_per_byte_labeled": 16.042083669862084,
      "perplexity_per_byte_ablated": 16.0960412085162
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7372735341389975,
      "eval_nll_ablated": 2.745177714029948,
      "label_use_gap": 0.007904179890950491,
      "perplexity_per_byte_labeled": 15.44481787005761,
      "perplexity_per_byte_ablated": 15.567380228248576
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.851742649078369,
      "eval_nll_ablated": 2.866512823104858,
      "label_use_gap": 0.014770174026489169,
      "perplexity_per_byte_labeled": 17.31793464290502,
      "perplexity_per_byte_ablated": 17.575621909551074
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.020454263687134,
      "eval_nll_ablated": 3.0229961395263674,
      "label_use_gap": 0.0025418758392334873,
      "perplexity_per_byte_labeled": 20.50060224847306,
      "perplexity_per_byte_ablated": 20.55277851872541
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.773329170544942,
      "eval_nll_ablated": 2.7786563873291015,
      "label_use_gap": 0.005327216784159372,
      "perplexity_per_byte_labeled": 16.011851560074877,
      "perplexity_per_byte_ablated": 16.097377770519092
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7325239896774294,
      "eval_nll_ablated": 2.7386685768763224,
      "label_use_gap": 0.006144587198892992,
      "perplexity_per_byte_labeled": 15.371635948848049,
      "perplexity_per_byte_ablated": 15.4663790869693
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8534378528594972,
      "eval_nll_ablated": 2.8706983089447022,
      "label_use_gap": 0.01726045608520499,
      "perplexity_per_byte_labeled": 17.3473169686715,
      "perplexity_per_byte_ablated": 17.649338588622335
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.0197593688964846,
      "eval_nll_ablated": 3.021921221415202,
      "label_use_gap": 0.0021618525187174775,
      "perplexity_per_byte_labeled": 20.486361435271924,
      "perplexity_per_byte_ablated": 20.53069783445067
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.774026648203532,
      "eval_nll_ablated": 2.779753065109253,
      "label_use_gap": 0.005726416905721177,
      "perplexity_per_byte_labeled": 16.023023364399744,
      "perplexity_per_byte_ablated": 16.11504109075383
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7356141805648804,
      "eval_nll_ablated": 2.7425416469573975,
      "label_use_gap": 0.006927466392517179,
      "perplexity_per_byte_labeled": 15.419210707867073,
      "perplexity_per_byte_ablated": 15.526397610020082
    },
    {
      "train_per_category": 30,
      "seed": 12101,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8517220338185627,
      "eval_nll_ablated": 2.8713234424591065,
      "label_use_gap": 0.019601408640543827,
      "perplexity_per_byte_labeled": 17.317577632862985,
      "perplexity_per_byte_ablated": 17.66037523100925
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9504988034566244,
      "eval_nll_ablated": 2.9504988034566244,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.115486221215978,
      "perplexity_per_byte_ablated": 19.115486221215978
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.776742617289225,
      "eval_nll_ablated": 2.776742617289225,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.06660055087524,
      "perplexity_per_byte_ablated": 16.06660055087524
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.73006272315979,
      "eval_nll_ablated": 2.73006272315979,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.333848777191701,
      "perplexity_per_byte_ablated": 15.333848777191701
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8409406979878744,
      "eval_nll_ablated": 2.8409406979878744,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 17.131873878701974,
      "perplexity_per_byte_ablated": 17.131873878701974
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9505974610646564,
      "eval_nll_ablated": 2.962269735336304,
      "label_use_gap": 0.01167227427164752,
      "perplexity_per_byte_labeled": 19.117372202394595,
      "perplexity_per_byte_ablated": 19.341822790211115
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7783212184906008,
      "eval_nll_ablated": 2.7869531949361166,
      "label_use_gap": 0.008631976445515832,
      "perplexity_per_byte_labeled": 16.091983335182654,
      "perplexity_per_byte_ablated": 16.231490200047208
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7409955660502114,
      "eval_nll_ablated": 2.754329466819763,
      "label_use_gap": 0.013333900769551654,
      "perplexity_per_byte_labeled": 15.502411090445868,
      "perplexity_per_byte_ablated": 15.710502956666508
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.866057364145915,
      "eval_nll_ablated": 2.8828062216440835,
      "label_use_gap": 0.01674885749816868,
      "perplexity_per_byte_labeled": 17.567618757785304,
      "perplexity_per_byte_ablated": 17.86433418687146
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9509067217508953,
      "eval_nll_ablated": 2.962714131673177,
      "label_use_gap": 0.011807409922281575,
      "perplexity_per_byte_labeled": 19.123285368348746,
      "perplexity_per_byte_ablated": 19.350420135580435
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.778771686553955,
      "eval_nll_ablated": 2.786623779932658,
      "label_use_gap": 0.007852093378703184,
      "perplexity_per_byte_labeled": 16.099233892700873,
      "perplexity_per_byte_ablated": 16.226144184224076
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7426665862401327,
      "eval_nll_ablated": 2.757157031695048,
      "label_use_gap": 0.014490445454915513,
      "perplexity_per_byte_labeled": 15.528337588188155,
      "perplexity_per_byte_ablated": 15.754988285938296
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.861395486195882,
      "eval_nll_ablated": 2.887171506881714,
      "label_use_gap": 0.025776020685831735,
      "perplexity_per_byte_labeled": 17.485911266419723,
      "perplexity_per_byte_ablated": 17.94248755795037
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.949216381708781,
      "eval_nll_ablated": 2.962942632039388,
      "label_use_gap": 0.013726250330607037,
      "perplexity_per_byte_labeled": 19.09098781796536,
      "perplexity_per_byte_ablated": 19.35484221887234
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.779141918818156,
      "eval_nll_ablated": 2.787591902414958,
      "label_use_gap": 0.008449983596801935,
      "perplexity_per_byte_labeled": 16.105195452029577,
      "perplexity_per_byte_ablated": 16.241860685731545
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7431589285532634,
      "eval_nll_ablated": 2.7614333391189576,
      "label_use_gap": 0.018274410565694232,
      "perplexity_per_byte_labeled": 15.535984728186222,
      "perplexity_per_byte_ablated": 15.822505719069516
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.867486524581909,
      "eval_nll_ablated": 2.908306121826172,
      "label_use_gap": 0.04081959724426287,
      "perplexity_per_byte_labeled": 17.592743652945153,
      "perplexity_per_byte_ablated": 18.325730706597614
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9504122416178387,
      "eval_nll_ablated": 2.962728500366211,
      "label_use_gap": 0.012316258748372277,
      "perplexity_per_byte_labeled": 19.11383162119303,
      "perplexity_per_byte_ablated": 19.35069817782499
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7793893337249758,
      "eval_nll_ablated": 2.7858025391896564,
      "label_use_gap": 0.006413205464680605,
      "perplexity_per_byte_labeled": 16.109180610435125,
      "perplexity_per_byte_ablated": 16.212824083771395
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7437295436859133,
      "eval_nll_ablated": 2.7607120275497437,
      "label_use_gap": 0.01698248386383039,
      "perplexity_per_byte_labeled": 15.544852325924865,
      "perplexity_per_byte_ablated": 15.811096877799656
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8733482996622723,
      "eval_nll_ablated": 2.9026676177978517,
      "label_use_gap": 0.02931931813557931,
      "perplexity_per_byte_labeled": 17.69617119763638,
      "perplexity_per_byte_ablated": 18.222691765918203
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.950503873825073,
      "eval_nll_ablated": 2.962099138895671,
      "label_use_gap": 0.011595265070597716,
      "perplexity_per_byte_labeled": 19.11558314401991,
      "perplexity_per_byte_ablated": 19.338523425525686
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.777966849009196,
      "eval_nll_ablated": 2.7862944920857746,
      "label_use_gap": 0.008327643076578628,
      "perplexity_per_byte_labeled": 16.086281837671415,
      "perplexity_per_byte_ablated": 16.220801991750232
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.740279801686605,
      "eval_nll_ablated": 2.7525507688522337,
      "label_use_gap": 0.012270967165628921,
      "perplexity_per_byte_labeled": 15.491318987177022,
      "perplexity_per_byte_ablated": 15.68258355444149
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8677969773610434,
      "eval_nll_ablated": 2.8882967948913576,
      "label_use_gap": 0.020499817530314157,
      "perplexity_per_byte_labeled": 17.59820621699503,
      "perplexity_per_byte_ablated": 17.96268938836953
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.949596818288167,
      "eval_nll_ablated": 2.9616149266560874,
      "label_use_gap": 0.012018108367920277,
      "perplexity_per_byte_labeled": 19.098252109781484,
      "perplexity_per_byte_ablated": 19.329161742491245
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.779203462600708,
      "eval_nll_ablated": 2.7859841426213583,
      "label_use_gap": 0.006780680020650465,
      "perplexity_per_byte_labeled": 16.10618665717738,
      "perplexity_per_byte_ablated": 16.215768655626672
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7410094022750853,
      "eval_nll_ablated": 2.7542520999908446,
      "label_use_gap": 0.013242697715759366,
      "perplexity_per_byte_labeled": 15.502625586775707,
      "perplexity_per_byte_ablated": 15.709287531889423
    },
    {
      "train_per_category": 30,
      "seed": 13103,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.872753302256266,
      "eval_nll_ablated": 2.900992488861084,
      "label_use_gap": 0.028239186604817945,
      "perplexity_per_byte_labeled": 17.685645153472567,
      "perplexity_per_byte_ablated": 18.192191960320795
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.944398546218872,
      "eval_nll_ablated": 2.944398546218872,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 18.99923178952681,
      "perplexity_per_byte_ablated": 18.99923178952681
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7793769677480062,
      "eval_nll_ablated": 2.7793769677480062,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.10898140591038,
      "perplexity_per_byte_ablated": 16.10898140591038
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7149551153182983,
      "eval_nll_ablated": 2.7149551153182983,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.103932121495003,
      "perplexity_per_byte_ablated": 15.103932121495003
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.801753123601278,
      "eval_nll_ablated": 2.801753123601278,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.47350155507725,
      "perplexity_per_byte_ablated": 16.47350155507725
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.951347510019938,
      "eval_nll_ablated": 2.954952526092529,
      "label_use_gap": 0.0036050160725911162,
      "perplexity_per_byte_labeled": 19.131716546250313,
      "perplexity_per_byte_ablated": 19.200811160663285
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7854222297668456,
      "eval_nll_ablated": 2.7902633428573607,
      "label_use_gap": 0.004841113090515048,
      "perplexity_per_byte_labeled": 16.206659366326335,
      "perplexity_per_byte_ablated": 16.285307856653304
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.728736996650696,
      "eval_nll_ablated": 2.7353028456370034,
      "label_use_gap": 0.006565848986307632,
      "perplexity_per_byte_labeled": 15.313533756437499,
      "perplexity_per_byte_ablated": 15.414410916223583
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8282601038614907,
      "eval_nll_ablated": 2.843603483835856,
      "label_use_gap": 0.015343379974365412,
      "perplexity_per_byte_labeled": 16.916003116794407,
      "perplexity_per_byte_ablated": 17.17755318012379
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.951644293467204,
      "eval_nll_ablated": 2.955159902572632,
      "label_use_gap": 0.0035156091054280303,
      "perplexity_per_byte_labeled": 19.137395365687144,
      "perplexity_per_byte_ablated": 19.204793370190927
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.78762629032135,
      "eval_nll_ablated": 2.79156756401062,
      "label_use_gap": 0.003941273689270108,
      "perplexity_per_byte_labeled": 16.242419218919782,
      "perplexity_per_byte_ablated": 16.306561356266513
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7303130626678467,
      "eval_nll_ablated": 2.7358820835749307,
      "label_use_gap": 0.005569020907083999,
      "perplexity_per_byte_labeled": 15.337687925876445,
      "perplexity_per_byte_ablated": 15.423342114211698
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8345963319142657,
      "eval_nll_ablated": 2.8432592868804933,
      "label_use_gap": 0.008662954966227598,
      "perplexity_per_byte_labeled": 17.02352705865514,
      "perplexity_per_byte_ablated": 17.171641736027503
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.950989007949829,
      "eval_nll_ablated": 2.9543307463328046,
      "label_use_gap": 0.0033417383829754677,
      "perplexity_per_byte_labeled": 19.12485901555667,
      "perplexity_per_byte_ablated": 19.18887619575761
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7869364976882935,
      "eval_nll_ablated": 2.7906636079152425,
      "label_use_gap": 0.0037271102269489376,
      "perplexity_per_byte_labeled": 16.23121918109544,
      "perplexity_per_byte_ablated": 16.291827601071066
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7298673152923585,
      "eval_nll_ablated": 2.733381175994873,
      "label_use_gap": 0.003513860702514471,
      "perplexity_per_byte_labeled": 15.330852715239203,
      "perplexity_per_byte_ablated": 15.38481795377011
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.833692741394043,
      "eval_nll_ablated": 2.8356836001078287,
      "label_use_gap": 0.0019908587137855704,
      "perplexity_per_byte_labeled": 17.008151708540627,
      "perplexity_per_byte_ablated": 17.04204626401538
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.951811742782593,
      "eval_nll_ablated": 2.9551905155181886,
      "label_use_gap": 0.0033787727355956143,
      "perplexity_per_byte_labeled": 19.140600177753765,
      "perplexity_per_byte_ablated": 19.2053812944838
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.788325484593709,
      "eval_nll_ablated": 2.7922324657440187,
      "label_use_gap": 0.00390698115030963,
      "perplexity_per_byte_labeled": 16.253779796569493,
      "perplexity_per_byte_ablated": 16.31740722249612
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7313632090886433,
      "eval_nll_ablated": 2.73578306833903,
      "label_use_gap": 0.004419859250386793,
      "perplexity_per_byte_labeled": 15.353803204175007,
      "perplexity_per_byte_ablated": 15.421815043956741
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8378753662109375,
      "eval_nll_ablated": 2.841293287277222,
      "label_use_gap": 0.0034179210662843573,
      "perplexity_per_byte_labeled": 17.07943940688676,
      "perplexity_per_byte_ablated": 17.137915458973712
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9511648654937743,
      "eval_nll_ablated": 2.9545076370239256,
      "label_use_gap": 0.0033427715301512784,
      "perplexity_per_byte_labeled": 19.128222562035287,
      "perplexity_per_byte_ablated": 19.192270829560425
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.785778117179871,
      "eval_nll_ablated": 2.7920254707336425,
      "label_use_gap": 0.006247353553771706,
      "perplexity_per_byte_labeled": 16.212428138857874,
      "perplexity_per_byte_ablated": 16.314029950170102
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.727441358566284,
      "eval_nll_ablated": 2.7356200138727824,
      "label_use_gap": 0.008178655306498417,
      "perplexity_per_byte_labeled": 15.293705806592937,
      "perplexity_per_byte_ablated": 15.419300653133076
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8319458325703937,
      "eval_nll_ablated": 2.841802549362183,
      "label_use_gap": 0.009856716791789122,
      "perplexity_per_byte_labeled": 16.978465954948764,
      "perplexity_per_byte_ablated": 17.14664537225048
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9514121214548745,
      "eval_nll_ablated": 2.954815562566121,
      "label_use_gap": 0.0034034411112466856,
      "perplexity_per_byte_labeled": 19.132952713844016,
      "perplexity_per_byte_ablated": 19.198181529942687
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.785261344909668,
      "eval_nll_ablated": 2.7900914986928305,
      "label_use_gap": 0.0048301537831623165,
      "perplexity_per_byte_labeled": 16.20405216998369,
      "perplexity_per_byte_ablated": 16.282509561972752
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.730169097582499,
      "eval_nll_ablated": 2.734783395131429,
      "label_use_gap": 0.0046142975489300575,
      "perplexity_per_byte_labeled": 15.335479993261577,
      "perplexity_per_byte_ablated": 15.406405971946192
    },
    {
      "train_per_category": 30,
      "seed": 14107,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8309339046478272,
      "eval_nll_ablated": 2.839913511276245,
      "label_use_gap": 0.008979606628417791,
      "perplexity_per_byte_labeled": 16.961293661193743,
      "perplexity_per_byte_ablated": 17.11428528044216
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.976511255900065,
      "eval_nll_ablated": 2.976511255900065,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.619250564900717,
      "perplexity_per_byte_ablated": 19.619250564900717
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.787291383743286,
      "eval_nll_ablated": 2.787291383743286,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.236980436672507,
      "perplexity_per_byte_ablated": 16.236980436672507
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7392370144526166,
      "eval_nll_ablated": 2.7392370144526166,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.475173257243164,
      "perplexity_per_byte_ablated": 15.475173257243164
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8439541816711427,
      "eval_nll_ablated": 2.8439541816711427,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 17.183578367287875,
      "perplexity_per_byte_ablated": 17.183578367287875
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.975959539413452,
      "eval_nll_ablated": 2.978702672322591,
      "label_use_gap": 0.002743132909138968,
      "perplexity_per_byte_labeled": 19.608429286322465,
      "perplexity_per_byte_ablated": 19.662291656038573
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.78756533463796,
      "eval_nll_ablated": 2.800072129567464,
      "label_use_gap": 0.012506794929504128,
      "perplexity_per_byte_labeled": 16.241429181330897,
      "perplexity_per_byte_ablated": 16.445832959134847
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7466566960016885,
      "eval_nll_ablated": 2.763105630874634,
      "label_use_gap": 0.016448934872945298,
      "perplexity_per_byte_labeled": 15.590421137297984,
      "perplexity_per_byte_ablated": 15.848987701566621
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8587023258209228,
      "eval_nll_ablated": 2.8750792344411216,
      "label_use_gap": 0.016376908620198805,
      "perplexity_per_byte_labeled": 17.438882259831008,
      "perplexity_per_byte_ablated": 17.72682864117798
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9771098295847573,
      "eval_nll_ablated": 2.980127731959025,
      "label_use_gap": 0.003017902374267578,
      "perplexity_per_byte_labeled": 19.630997647398743,
      "perplexity_per_byte_ablated": 19.690331568766023
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.78767991065979,
      "eval_nll_ablated": 2.79809414545695,
      "label_use_gap": 0.01041423479715986,
      "perplexity_per_byte_labeled": 16.243290166285423,
      "perplexity_per_byte_ablated": 16.413335513168693
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7465017398198444,
      "eval_nll_ablated": 2.7629048029581704,
      "label_use_gap": 0.01640306313832607,
      "perplexity_per_byte_labeled": 15.5880054923296,
      "perplexity_per_byte_ablated": 15.84580510197658
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8553046385447183,
      "eval_nll_ablated": 2.877611287434896,
      "label_use_gap": 0.022306648890177527,
      "perplexity_per_byte_labeled": 17.379730937218593,
      "perplexity_per_byte_ablated": 17.77177078464202
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.976777950922648,
      "eval_nll_ablated": 2.979055817921956,
      "label_use_gap": 0.002277866999308209,
      "perplexity_per_byte_labeled": 19.62448361915693,
      "perplexity_per_byte_ablated": 19.669236534014708
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7877398490905763,
      "eval_nll_ablated": 2.7949870745340983,
      "label_use_gap": 0.007247225443522076,
      "perplexity_per_byte_labeled": 16.244263792787326,
      "perplexity_per_byte_ablated": 16.36241725992403
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.749489196141561,
      "eval_nll_ablated": 2.756948971748352,
      "label_use_gap": 0.007459775606791119,
      "perplexity_per_byte_labeled": 15.634643607860463,
      "perplexity_per_byte_ablated": 15.751710644900058
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8612002690633136,
      "eval_nll_ablated": 2.878063376744588,
      "label_use_gap": 0.016863107681274414,
      "perplexity_per_byte_labeled": 17.48249805013193,
      "perplexity_per_byte_ablated": 17.779807028641184
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9768510023752848,
      "eval_nll_ablated": 2.9792705217997235,
      "label_use_gap": 0.002419519424438743,
      "perplexity_per_byte_labeled": 19.625917268557004,
      "perplexity_per_byte_ablated": 19.673460048757555
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.788621425628662,
      "eval_nll_ablated": 2.799297340710958,
      "label_use_gap": 0.010675915082295706,
      "perplexity_per_byte_labeled": 16.258590668816534,
      "perplexity_per_byte_ablated": 16.433095845946003
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.750778388977051,
      "eval_nll_ablated": 2.7609794537226358,
      "label_use_gap": 0.010201064745584798,
      "perplexity_per_byte_labeled": 15.654812676501106,
      "perplexity_per_byte_ablated": 15.815325744356509
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8552942117055258,
      "eval_nll_ablated": 2.8753231525421143,
      "label_use_gap": 0.02002894083658857,
      "perplexity_per_byte_labeled": 17.379549722503654,
      "perplexity_per_byte_ablated": 17.731153062937693
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9768469174702963,
      "eval_nll_ablated": 2.9798158009847007,
      "label_use_gap": 0.0029688835144043857,
      "perplexity_per_byte_labeled": 19.625837098713394,
      "perplexity_per_byte_ablated": 19.68419050229923
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7866130034128824,
      "eval_nll_ablated": 2.7990025679270425,
      "label_use_gap": 0.012389564514160067,
      "perplexity_per_byte_labeled": 16.225969323802587,
      "perplexity_per_byte_ablated": 16.42825253040902
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7452900568644205,
      "eval_nll_ablated": 2.7634385029474893,
      "label_use_gap": 0.01814844608306876,
      "perplexity_per_byte_labeled": 15.56912921010968,
      "perplexity_per_byte_ablated": 15.854264265117104
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8554774284362794,
      "eval_nll_ablated": 2.869248390197754,
      "label_use_gap": 0.01377096176147452,
      "perplexity_per_byte_labeled": 17.382734238505176,
      "perplexity_per_byte_ablated": 17.623767024556365
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.976818084716797,
      "eval_nll_ablated": 2.9791756788889567,
      "label_use_gap": 0.0023575941721598603,
      "perplexity_per_byte_labeled": 19.62527123994778,
      "perplexity_per_byte_ablated": 19.671594249022014
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.787608750661214,
      "eval_nll_ablated": 2.797058582305908,
      "label_use_gap": 0.00944983164469404,
      "perplexity_per_byte_labeled": 16.242134334905288,
      "perplexity_per_byte_ablated": 16.396347265445414
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7494439760843914,
      "eval_nll_ablated": 2.7533490657806396,
      "label_use_gap": 0.003905089696248254,
      "perplexity_per_byte_labeled": 15.633936624367728,
      "perplexity_per_byte_ablated": 15.695107911135084
    },
    {
      "train_per_category": 30,
      "seed": 15109,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8542011260986326,
      "eval_nll_ablated": 2.8723405679066976,
      "label_use_gap": 0.018139441808064927,
      "perplexity_per_byte_labeled": 17.360562765922705,
      "perplexity_per_byte_ablated": 17.6783471863879
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.946763547261556,
      "eval_nll_ablated": 2.946763547261556,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.044218167966932,
      "perplexity_per_byte_ablated": 19.044218167966932
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7914231618245444,
      "eval_nll_ablated": 2.7914231618245444,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 16.304206823163245,
      "perplexity_per_byte_ablated": 16.304206823163245
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7377599318822226,
      "eval_nll_ablated": 2.7377599318822226,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.452332021899233,
      "perplexity_per_byte_ablated": 15.452332021899233
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8392866134643553,
      "eval_nll_ablated": 2.8392866134643553,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 17.10355973470963,
      "perplexity_per_byte_ablated": 17.10355973470963
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9557276884714763,
      "eval_nll_ablated": 2.9593454360961915,
      "label_use_gap": 0.0036177476247152285,
      "perplexity_per_byte_labeled": 19.215700677271293,
      "perplexity_per_byte_ablated": 19.28534413301869
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7938737233479816,
      "eval_nll_ablated": 2.8181073824564615,
      "label_use_gap": 0.024233659108479966,
      "perplexity_per_byte_labeled": 16.344210280521718,
      "perplexity_per_byte_ablated": 16.74512853863471
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7398926337560017,
      "eval_nll_ablated": 2.780439281463623,
      "label_use_gap": 0.040546647707621286,
      "perplexity_per_byte_labeled": 15.485322406179279,
      "perplexity_per_byte_ablated": 16.12610329059878
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8375749429066977,
      "eval_nll_ablated": 2.863423252105713,
      "label_use_gap": 0.02584830919901515,
      "perplexity_per_byte_labeled": 17.074309115933644,
      "perplexity_per_byte_ablated": 17.52140457508688
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.954886547724406,
      "eval_nll_ablated": 2.95840744972229,
      "label_use_gap": 0.0035209019978839073,
      "perplexity_per_byte_labeled": 19.199544364267304,
      "perplexity_per_byte_ablated": 19.267263224154643
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.794716993967692,
      "eval_nll_ablated": 2.819425519307454,
      "label_use_gap": 0.02470852533976231,
      "perplexity_per_byte_labeled": 16.357998685715064,
      "perplexity_per_byte_ablated": 16.767215463232738
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7443572282791138,
      "eval_nll_ablated": 2.769418239593506,
      "label_use_gap": 0.02506101131439209,
      "perplexity_per_byte_labeled": 15.554612653116472,
      "perplexity_per_byte_ablated": 15.949352608423087
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.852420171101888,
      "eval_nll_ablated": 2.888509766260783,
      "label_use_gap": 0.03608959515889465,
      "perplexity_per_byte_labeled": 17.329671900703794,
      "perplexity_per_byte_ablated": 17.966515334321144
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.956347878774007,
      "eval_nll_ablated": 2.958966827392578,
      "label_use_gap": 0.0026189486185712596,
      "perplexity_per_byte_labeled": 19.227621764777,
      "perplexity_per_byte_ablated": 19.278043915927817
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.795396423339844,
      "eval_nll_ablated": 2.808971118927002,
      "label_use_gap": 0.013574695587158292,
      "perplexity_per_byte_labeled": 16.369116566971613,
      "perplexity_per_byte_ablated": 16.592837376891627
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7398658116658527,
      "eval_nll_ablated": 2.758923554420471,
      "label_use_gap": 0.019057742754618356,
      "perplexity_per_byte_labeled": 15.484907063035925,
      "perplexity_per_byte_ablated": 15.782844427793997
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8454519271850587,
      "eval_nll_ablated": 2.873229185740153,
      "label_use_gap": 0.027777258555094164,
      "perplexity_per_byte_labeled": 17.20933427778548,
      "perplexity_per_byte_ablated": 17.694063462811304
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.956392208735148,
      "eval_nll_ablated": 2.958884572982788,
      "label_use_gap": 0.002492364247640211,
      "perplexity_per_byte_labeled": 19.22847414339548,
      "perplexity_per_byte_ablated": 19.276458277017397
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7948152860005697,
      "eval_nll_ablated": 2.81172997156779,
      "label_use_gap": 0.0169146855672202,
      "perplexity_per_byte_labeled": 16.359606625682243,
      "perplexity_per_byte_ablated": 16.63867777448271
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7408514579137164,
      "eval_nll_ablated": 2.766344952583313,
      "label_use_gap": 0.025493494669596384,
      "perplexity_per_byte_labeled": 15.500177227835199,
      "perplexity_per_byte_ablated": 15.900410914688003
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8521873315175372,
      "eval_nll_ablated": 2.8867926597595215,
      "label_use_gap": 0.03460532824198426,
      "perplexity_per_byte_labeled": 17.325637336822812,
      "perplexity_per_byte_ablated": 17.935691385610518
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.955455207824707,
      "eval_nll_ablated": 2.95845513343811,
      "label_use_gap": 0.0029999256134032315,
      "perplexity_per_byte_labeled": 19.210465483999442,
      "perplexity_per_byte_ablated": 19.26818198076355
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.794100220998128,
      "eval_nll_ablated": 2.8096592903137205,
      "label_use_gap": 0.015559069315592478,
      "perplexity_per_byte_labeled": 16.347912625014093,
      "perplexity_per_byte_ablated": 16.60426002271709
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.7423243045806887,
      "eval_nll_ablated": 2.771181670824687,
      "label_use_gap": 0.02885736624399815,
      "perplexity_per_byte_labeled": 15.52302343255095,
      "perplexity_per_byte_ablated": 15.977503008282122
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.844036324818929,
      "eval_nll_ablated": 2.8824827671051025,
      "label_use_gap": 0.03844644228617344,
      "perplexity_per_byte_labeled": 17.184989938479855,
      "perplexity_per_byte_ablated": 17.858556821300773
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9553633371988934,
      "eval_nll_ablated": 2.9583911101023355,
      "label_use_gap": 0.0030277729034420275,
      "perplexity_per_byte_labeled": 19.208700687580976,
      "perplexity_per_byte_ablated": 19.266948406968
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.794232392311096,
      "eval_nll_ablated": 2.8106375217437742,
      "label_use_gap": 0.016405129432678134,
      "perplexity_per_byte_labeled": 16.350073492889255,
      "perplexity_per_byte_ablated": 16.620510778948436
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.741923522949219,
      "eval_nll_ablated": 2.7654433965682985,
      "label_use_gap": 0.02351987361907959,
      "perplexity_per_byte_labeled": 15.516803336427705,
      "perplexity_per_byte_ablated": 15.886082263597988
    },
    {
      "train_per_category": 30,
      "seed": 16111,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.8491083780924478,
      "eval_nll_ablated": 2.88928337097168,
      "label_use_gap": 0.040174992879232096,
      "perplexity_per_byte_labeled": 17.272374545307322,
      "perplexity_per_byte_ablated": 17.980419692767338
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9922436396280925,
      "eval_nll_ablated": 2.9922436396280925,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.93034888574323,
      "perplexity_per_byte_ablated": 19.93034888574323
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.743067256609599,
      "eval_nll_ablated": 2.743067256609599,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.53456057954768,
      "perplexity_per_byte_ablated": 15.53456057954768
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.627923019727071,
      "eval_nll_ablated": 2.627923019727071,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.844984259685848,
      "perplexity_per_byte_ablated": 13.844984259685848
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5673970063527425,
      "eval_nll_ablated": 2.5673970063527425,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.031858408779868,
      "perplexity_per_byte_ablated": 13.031858408779868
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.0052895228068035,
      "eval_nll_ablated": 3.0065565427144367,
      "label_use_gap": 0.0012670199076332267,
      "perplexity_per_byte_labeled": 20.19206131205112,
      "perplexity_per_byte_ablated": 20.21766127011317
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7436350425084433,
      "eval_nll_ablated": 2.749506870905558,
      "label_use_gap": 0.005871828397114864,
      "perplexity_per_byte_labeled": 15.543383388485719,
      "perplexity_per_byte_ablated": 15.634919948938537
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6296698411305743,
      "eval_nll_ablated": 2.6383867899576825,
      "label_use_gap": 0.008716948827108162,
      "perplexity_per_byte_labeled": 13.869190110015442,
      "perplexity_per_byte_ablated": 13.9906155914546
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.571339511871338,
      "eval_nll_ablated": 2.582137091954549,
      "label_use_gap": 0.010797580083211411,
      "perplexity_per_byte_labeled": 13.08333799507014,
      "perplexity_per_byte_ablated": 13.225371815656338
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.0063373406728107,
      "eval_nll_ablated": 3.007396141688029,
      "label_use_gap": 0.0010588010152181582,
      "perplexity_per_byte_labeled": 20.213230003174978,
      "perplexity_per_byte_ablated": 20.234643125740686
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.743599025408427,
      "eval_nll_ablated": 2.749717092514038,
      "label_use_gap": 0.006118067105611136,
      "perplexity_per_byte_labeled": 15.542823570973187,
      "perplexity_per_byte_ablated": 15.638207092460853
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.629477016131083,
      "eval_nll_ablated": 2.6377145051956177,
      "label_use_gap": 0.008237489064534476,
      "perplexity_per_byte_labeled": 13.866516041261473,
      "perplexity_per_byte_ablated": 13.981213074719058
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.572112043698629,
      "eval_nll_ablated": 2.583160392443339,
      "label_use_gap": 0.011048348744710168,
      "perplexity_per_byte_labeled": 13.093449195187622,
      "perplexity_per_byte_ablated": 13.238912271895904
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.0081170399983725,
      "eval_nll_ablated": 3.0095998922983807,
      "label_use_gap": 0.0014828523000081084,
      "perplexity_per_byte_labeled": 20.249235504959238,
      "perplexity_per_byte_ablated": 20.279284403934565
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7435245911280313,
      "eval_nll_ablated": 2.7493104139963784,
      "label_use_gap": 0.005785822868347168,
      "perplexity_per_byte_labeled": 15.541666695141508,
      "perplexity_per_byte_ablated": 15.631848662587688
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6293002049128216,
      "eval_nll_ablated": 2.637801170349121,
      "label_use_gap": 0.008500965436299524,
      "perplexity_per_byte_labeled": 13.864064502403346,
      "perplexity_per_byte_ablated": 13.982424811203249
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.568268402417501,
      "eval_nll_ablated": 2.5791052977244058,
      "label_use_gap": 0.010836895306904903,
      "perplexity_per_byte_labeled": 13.04321926809893,
      "perplexity_per_byte_ablated": 13.185335930643458
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.008435328801473,
      "eval_nll_ablated": 3.0097570419311523,
      "label_use_gap": 0.001321713129679214,
      "perplexity_per_byte_labeled": 20.255681635703013,
      "perplexity_per_byte_ablated": 20.282471536453308
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.743811543782552,
      "eval_nll_ablated": 2.7498019218444822,
      "label_use_gap": 0.005990378061930368,
      "perplexity_per_byte_labeled": 15.546127057581169,
      "perplexity_per_byte_ablated": 15.639533727365832
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6293943881988526,
      "eval_nll_ablated": 2.637298591931661,
      "label_use_gap": 0.007904203732808313,
      "perplexity_per_byte_labeled": 13.865370327048392,
      "perplexity_per_byte_ablated": 13.975399311849431
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5701894919077555,
      "eval_nll_ablated": 2.582503151893616,
      "label_use_gap": 0.012313659985860426,
      "perplexity_per_byte_labeled": 13.068300543527561,
      "perplexity_per_byte_ablated": 13.230213980664555
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.006278117497762,
      "eval_nll_ablated": 3.007923650741577,
      "label_use_gap": 0.0016455332438147785,
      "perplexity_per_byte_labeled": 20.21203294696329,
      "perplexity_per_byte_ablated": 20.245319898984523
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7440849939982095,
      "eval_nll_ablated": 2.749976062774658,
      "label_use_gap": 0.005891068776448627,
      "perplexity_per_byte_labeled": 15.550378730661672,
      "perplexity_per_byte_ablated": 15.642257447465326
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6295252005259195,
      "eval_nll_ablated": 2.639562710126241,
      "label_use_gap": 0.01003750960032157,
      "perplexity_per_byte_labeled": 13.867184207042863,
      "perplexity_per_byte_ablated": 14.007077115321204
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.573770602544149,
      "eval_nll_ablated": 2.5851007143656415,
      "label_use_gap": 0.011330111821492572,
      "perplexity_per_byte_labeled": 13.115183469973264,
      "perplexity_per_byte_ablated": 13.26462496098463
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 3.008597167332967,
      "eval_nll_ablated": 3.00959734916687,
      "label_use_gap": 0.0010001818339029356,
      "perplexity_per_byte_labeled": 20.258960050753124,
      "perplexity_per_byte_ablated": 20.27923283111296
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.743721922238668,
      "eval_nll_ablated": 2.750733574231466,
      "label_use_gap": 0.0070116519927978516,
      "perplexity_per_byte_labeled": 15.544733852104397,
      "perplexity_per_byte_ablated": 15.654111125773852
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.629333217938741,
      "eval_nll_ablated": 2.6385668992996214,
      "label_use_gap": 0.009233681360880208,
      "perplexity_per_byte_labeled": 13.864522204679144,
      "perplexity_per_byte_ablated": 13.993135658959137
    },
    {
      "train_per_category": 100,
      "seed": 12101,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.570802609125773,
      "eval_nll_ablated": 2.5831378221511843,
      "label_use_gap": 0.012335213025411296,
      "perplexity_per_byte_labeled": 13.076315400373318,
      "perplexity_per_byte_ablated": 13.238613469150161
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.92883194287618,
      "eval_nll_ablated": 2.92883194287618,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 18.705768325077297,
      "perplexity_per_byte_ablated": 18.705768325077297
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.734233832359314,
      "eval_nll_ablated": 2.734233832359314,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.397941510861305,
      "perplexity_per_byte_ablated": 15.397941510861305
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.628937514623006,
      "eval_nll_ablated": 2.628937514623006,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.859037052589676,
      "perplexity_per_byte_ablated": 13.859037052589676
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.559117158253988,
      "eval_nll_ablated": 2.559117158253988,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.924402075664709,
      "perplexity_per_byte_ablated": 12.924402075664709
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9296617190043133,
      "eval_nll_ablated": 2.943746566772461,
      "label_use_gap": 0.014084847768147668,
      "perplexity_per_byte_labeled": 18.721296366599965,
      "perplexity_per_byte_ablated": 18.986848718095114
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.736471430460612,
      "eval_nll_ablated": 2.748578174908956,
      "label_use_gap": 0.012106744448344031,
      "perplexity_per_byte_labeled": 15.432434491872563,
      "perplexity_per_byte_ablated": 15.620406601661637
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6262181361516315,
      "eval_nll_ablated": 2.635526951154073,
      "label_use_gap": 0.009308815002441406,
      "perplexity_per_byte_labeled": 13.821400283098939,
      "perplexity_per_byte_ablated": 13.950661843969211
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5443305810292562,
      "eval_nll_ablated": 2.5563185850779218,
      "label_use_gap": 0.011988004048665513,
      "perplexity_per_byte_labeled": 12.73470038205807,
      "perplexity_per_byte_ablated": 12.88828275555317
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.930577262242635,
      "eval_nll_ablated": 2.944659392038981,
      "label_use_gap": 0.014082129796345821,
      "perplexity_per_byte_labeled": 18.738444371573213,
      "perplexity_per_byte_ablated": 19.004188306139607
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.736658978462219,
      "eval_nll_ablated": 2.7480763753255206,
      "label_use_gap": 0.011417396863301565,
      "perplexity_per_byte_labeled": 15.435329085550592,
      "perplexity_per_byte_ablated": 15.612570254438154
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.626683982213338,
      "eval_nll_ablated": 2.635181943575541,
      "label_use_gap": 0.008497961362202933,
      "perplexity_per_byte_labeled": 13.82784042792968,
      "perplexity_per_byte_ablated": 13.945849590087269
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.545380187034607,
      "eval_nll_ablated": 2.556603678067525,
      "label_use_gap": 0.011223491032918087,
      "perplexity_per_byte_labeled": 12.748073817246603,
      "perplexity_per_byte_ablated": 12.891957638431588
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9286059379577636,
      "eval_nll_ablated": 2.9433672269185385,
      "label_use_gap": 0.014761288960774888,
      "perplexity_per_byte_labeled": 18.70154120712579,
      "perplexity_per_byte_ablated": 18.97964761559488
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7364362398783366,
      "eval_nll_ablated": 2.747935485839844,
      "label_use_gap": 0.01149924596150731,
      "perplexity_per_byte_labeled": 15.431891425072342,
      "perplexity_per_byte_ablated": 15.610370762391204
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.626144790649414,
      "eval_nll_ablated": 2.6341784874598186,
      "label_use_gap": 0.008033696810404667,
      "perplexity_per_byte_labeled": 13.820386582729462,
      "perplexity_per_byte_ablated": 13.931862560885804
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5456422408421835,
      "eval_nll_ablated": 2.5584750334421793,
      "label_use_gap": 0.012832792599995813,
      "perplexity_per_byte_labeled": 12.75141493628704,
      "perplexity_per_byte_ablated": 12.91610566036626
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9296502590179445,
      "eval_nll_ablated": 2.943139139811198,
      "label_use_gap": 0.01348888079325361,
      "perplexity_per_byte_labeled": 18.72108182202814,
      "perplexity_per_byte_ablated": 18.975319096330388
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.736702060699463,
      "eval_nll_ablated": 2.7471901337305704,
      "label_use_gap": 0.010488073031107437,
      "perplexity_per_byte_labeled": 15.435994088384996,
      "perplexity_per_byte_ablated": 15.59873987470871
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6268378655115763,
      "eval_nll_ablated": 2.634275452295939,
      "label_use_gap": 0.007437586784362793,
      "perplexity_per_byte_labeled": 13.82996846535275,
      "perplexity_per_byte_ablated": 13.933213527152926
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.545373217264811,
      "eval_nll_ablated": 2.557042956352234,
      "label_use_gap": 0.011669739087422748,
      "perplexity_per_byte_labeled": 12.74798496641639,
      "perplexity_per_byte_ablated": 12.897622039502643
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.930468289057414,
      "eval_nll_ablated": 2.945099449157715,
      "label_use_gap": 0.014631160100301077,
      "perplexity_per_byte_labeled": 18.736402494860876,
      "perplexity_per_byte_ablated": 19.012553074842504
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.73629576365153,
      "eval_nll_ablated": 2.748137847582499,
      "label_use_gap": 0.011842083930968972,
      "perplexity_per_byte_labeled": 15.42972376344849,
      "perplexity_per_byte_ablated": 15.613530023868224
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.626846933364868,
      "eval_nll_ablated": 2.635247747103373,
      "label_use_gap": 0.008400813738504986,
      "perplexity_per_byte_labeled": 13.830093874046417,
      "perplexity_per_byte_ablated": 13.946767306383066
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5444413661956786,
      "eval_nll_ablated": 2.557741721471151,
      "label_use_gap": 0.013300355275472242,
      "perplexity_per_byte_labeled": 12.736111276110858,
      "perplexity_per_byte_ablated": 12.906637597412683
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9291786193847655,
      "eval_nll_ablated": 2.9431432406107585,
      "label_use_gap": 0.013964621225992957,
      "perplexity_per_byte_labeled": 18.71225429973321,
      "perplexity_per_byte_ablated": 18.975396910470145
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7368439277013143,
      "eval_nll_ablated": 2.7487276871999105,
      "label_use_gap": 0.011883759498596103,
      "perplexity_per_byte_labeled": 15.438184101928565,
      "perplexity_per_byte_ablated": 15.622742219035693
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.62659375667572,
      "eval_nll_ablated": 2.6355981667836508,
      "label_use_gap": 0.009004410107930738,
      "perplexity_per_byte_labeled": 13.826592859875069,
      "perplexity_per_byte_ablated": 13.951655384512845
    },
    {
      "train_per_category": 100,
      "seed": 13103,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5443597078323363,
      "eval_nll_ablated": 2.55556534131368,
      "label_use_gap": 0.011205633481343735,
      "perplexity_per_byte_labeled": 12.735071308570308,
      "perplexity_per_byte_ablated": 12.878578392270168
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9241812388102213,
      "eval_nll_ablated": 2.9241812388102213,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 18.61897531301549,
      "perplexity_per_byte_ablated": 18.61897531301549
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.742313528060913,
      "eval_nll_ablated": 2.742313528060913,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.522856149283317,
      "perplexity_per_byte_ablated": 15.522856149283317
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6367952903111775,
      "eval_nll_ablated": 2.6367952903111775,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.968367240503769,
      "perplexity_per_byte_ablated": 13.968367240503769
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5712632735570273,
      "eval_nll_ablated": 2.5712632735570273,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.082340581456892,
      "perplexity_per_byte_ablated": 13.082340581456892
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.931199057896932,
      "eval_nll_ablated": 2.935377899805705,
      "label_use_gap": 0.0041788419087729345,
      "perplexity_per_byte_labeled": 18.750099478023493,
      "perplexity_per_byte_ablated": 18.828617121664898
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7445154587427774,
      "eval_nll_ablated": 2.751272130012512,
      "label_use_gap": 0.006756671269734671,
      "perplexity_per_byte_labeled": 15.557074061418435,
      "perplexity_per_byte_ablated": 15.662544008402797
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.638077219327291,
      "eval_nll_ablated": 2.642395822207133,
      "label_use_gap": 0.004318602879842093,
      "perplexity_per_byte_labeled": 13.986285178085371,
      "perplexity_per_byte_ablated": 14.046817001909906
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.574039363861084,
      "eval_nll_ablated": 2.5819741090138755,
      "label_use_gap": 0.007934745152791578,
      "perplexity_per_byte_labeled": 13.118708797669134,
      "perplexity_per_byte_ablated": 13.223216481312456
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.930969301859538,
      "eval_nll_ablated": 2.935598389307658,
      "label_use_gap": 0.004629087448119851,
      "perplexity_per_byte_labeled": 18.745792024317375,
      "perplexity_per_byte_ablated": 18.832769091792606
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7440528233846027,
      "eval_nll_ablated": 2.7506850481033327,
      "label_use_gap": 0.00663222471873004,
      "perplexity_per_byte_labeled": 15.54987847348292,
      "perplexity_per_byte_ablated": 15.653351510802288
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.638905628522237,
      "eval_nll_ablated": 2.6437251885732014,
      "label_use_gap": 0.004819560050964178,
      "perplexity_per_byte_labeled": 13.997876345782107,
      "perplexity_per_byte_ablated": 14.065502785356765
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.575141366322835,
      "eval_nll_ablated": 2.5826502164204914,
      "label_use_gap": 0.007508850097656339,
      "perplexity_per_byte_labeled": 13.133173615727866,
      "perplexity_per_byte_ablated": 13.232159818902467
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.930930582682292,
      "eval_nll_ablated": 2.9351831436157227,
      "label_use_gap": 0.00425256093343096,
      "perplexity_per_byte_labeled": 18.74506621672479,
      "perplexity_per_byte_ablated": 18.824950488992943
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7440354267756146,
      "eval_nll_ablated": 2.7510531346003213,
      "label_use_gap": 0.007017707824706676,
      "perplexity_per_byte_labeled": 15.549607960680316,
      "perplexity_per_byte_ablated": 15.659114358674207
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.637821388244629,
      "eval_nll_ablated": 2.6421560923258465,
      "label_use_gap": 0.004334704081217389,
      "perplexity_per_byte_labeled": 13.9827075092648,
      "perplexity_per_byte_ablated": 14.043449963743539
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.574793306986491,
      "eval_nll_ablated": 2.5823572079340615,
      "label_use_gap": 0.007563900947570534,
      "perplexity_per_byte_labeled": 13.128603287453938,
      "perplexity_per_byte_ablated": 13.228283251743004
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.931461747487386,
      "eval_nll_ablated": 2.9362136840820314,
      "label_use_gap": 0.004751936594645567,
      "perplexity_per_byte_labeled": 18.755025580966,
      "perplexity_per_byte_ablated": 18.84436036186003
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.744964599609375,
      "eval_nll_ablated": 2.752029824256897,
      "label_use_gap": 0.007065224647521795,
      "perplexity_per_byte_labeled": 15.564062948524022,
      "perplexity_per_byte_ablated": 15.674415924923824
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.638263948758443,
      "eval_nll_ablated": 2.643217746416728,
      "label_use_gap": 0.004953797658284742,
      "perplexity_per_byte_labeled": 13.988897073011845,
      "perplexity_per_byte_ablated": 14.058367166901405
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.575982705752055,
      "eval_nll_ablated": 2.586537718772888,
      "label_use_gap": 0.010555013020833393,
      "perplexity_per_byte_labeled": 13.144227721997241,
      "perplexity_per_byte_ablated": 13.28369998768029
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9312453587849934,
      "eval_nll_ablated": 2.935544013977051,
      "label_use_gap": 0.00429865519205741,
      "perplexity_per_byte_labeled": 18.750967644378846,
      "perplexity_per_byte_ablated": 18.831745081587695
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7442485014597575,
      "eval_nll_ablated": 2.7515228509902956,
      "label_use_gap": 0.007274349530538071,
      "perplexity_per_byte_labeled": 15.552921541492639,
      "perplexity_per_byte_ablated": 15.666471429073965
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6381590684254963,
      "eval_nll_ablated": 2.643572775522868,
      "label_use_gap": 0.005413707097371656,
      "perplexity_per_byte_labeled": 13.987429989764703,
      "perplexity_per_byte_ablated": 14.063359182533448
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5759990294774373,
      "eval_nll_ablated": 2.5854902029037476,
      "label_use_gap": 0.00949117342631034,
      "perplexity_per_byte_labeled": 13.14444228651218,
      "perplexity_per_byte_ablated": 13.269792386630415
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.931464449564616,
      "eval_nll_ablated": 2.936136833826701,
      "label_use_gap": 0.00467238426208505,
      "perplexity_per_byte_labeled": 18.755076258562035,
      "perplexity_per_byte_ablated": 18.84291222360029
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7442235151926675,
      "eval_nll_ablated": 2.7519275665283205,
      "label_use_gap": 0.007704051335652995,
      "perplexity_per_byte_labeled": 15.552532936895883,
      "perplexity_per_byte_ablated": 15.672813176702673
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.638070797920227,
      "eval_nll_ablated": 2.6428736209869386,
      "label_use_gap": 0.004802823066711426,
      "perplexity_per_byte_labeled": 13.98619536674329,
      "perplexity_per_byte_ablated": 14.053530157574649
    },
    {
      "train_per_category": 100,
      "seed": 14107,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5752899487813314,
      "eval_nll_ablated": 2.5852940638860065,
      "label_use_gap": 0.010004115104675027,
      "perplexity_per_byte_labeled": 13.135125119927608,
      "perplexity_per_byte_ablated": 13.267189917817253
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9629910310109455,
      "eval_nll_ablated": 2.9629910310109455,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 19.355778995999735,
      "perplexity_per_byte_ablated": 19.355778995999735
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7566370407740277,
      "eval_nll_ablated": 2.7566370407740277,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.746797964699715,
      "perplexity_per_byte_ablated": 15.746797964699715
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.636942505836487,
      "eval_nll_ablated": 2.636942505836487,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.970423752396318,
      "perplexity_per_byte_ablated": 13.970423752396318
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5701453844706217,
      "eval_nll_ablated": 2.5701453844706217,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.067724146994674,
      "perplexity_per_byte_ablated": 13.067724146994674
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9625293413798013,
      "eval_nll_ablated": 2.9649180730183917,
      "label_use_gap": 0.0023887316385904356,
      "perplexity_per_byte_labeled": 19.34684469613007,
      "perplexity_per_byte_ablated": 19.39311435706273
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7532084226608275,
      "eval_nll_ablated": 2.7569517532984418,
      "label_use_gap": 0.0037433306376142284,
      "perplexity_per_byte_labeled": 15.692900657413837,
      "perplexity_per_byte_ablated": 15.751754459133151
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6386457045873004,
      "eval_nll_ablated": 2.6487537463506063,
      "label_use_gap": 0.01010804176330593,
      "perplexity_per_byte_labeled": 13.994238435492012,
      "perplexity_per_byte_ablated": 14.136410110155165
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.577270778020223,
      "eval_nll_ablated": 2.58638645807902,
      "label_use_gap": 0.00911568005879726,
      "perplexity_per_byte_labeled": 13.161169345888023,
      "perplexity_per_byte_ablated": 13.28169083795954
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.962199417750041,
      "eval_nll_ablated": 2.964713621139526,
      "label_use_gap": 0.002514203389485381,
      "perplexity_per_byte_labeled": 19.34046276773589,
      "perplexity_per_byte_ablated": 19.389149803689374
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7529099305470783,
      "eval_nll_ablated": 2.757125488917033,
      "label_use_gap": 0.004215558369954575,
      "perplexity_per_byte_labeled": 15.688217149355633,
      "perplexity_per_byte_ablated": 15.754491337677765
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6388718843460084,
      "eval_nll_ablated": 2.6564948002497357,
      "label_use_gap": 0.017622915903727332,
      "perplexity_per_byte_labeled": 13.997404006945253,
      "perplexity_per_byte_ablated": 14.24626547231814
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.577949047088623,
      "eval_nll_ablated": 2.5882378101348875,
      "label_use_gap": 0.01028876304626447,
      "perplexity_per_byte_labeled": 13.170099188034825,
      "perplexity_per_byte_ablated": 13.306302699177952
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9630980332692465,
      "eval_nll_ablated": 2.9653693199157716,
      "label_use_gap": 0.0022712866465250947,
      "perplexity_per_byte_labeled": 19.35785021887427,
      "perplexity_per_byte_ablated": 19.40186741449336
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.754714918136597,
      "eval_nll_ablated": 2.758142669995626,
      "label_use_gap": 0.0034277518590291045,
      "perplexity_per_byte_labeled": 15.716559757945301,
      "perplexity_per_byte_ablated": 15.77052466118354
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6397120157877603,
      "eval_nll_ablated": 2.6519572496414185,
      "label_use_gap": 0.012245233853658277,
      "perplexity_per_byte_labeled": 14.009168607367773,
      "perplexity_per_byte_ablated": 14.18176876096605
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5779773871103924,
      "eval_nll_ablated": 2.5860133806864423,
      "label_use_gap": 0.008035993576049894,
      "perplexity_per_byte_labeled": 13.170472434221397,
      "perplexity_per_byte_ablated": 13.276736663575383
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9630178769429523,
      "eval_nll_ablated": 2.9655383904774983,
      "label_use_gap": 0.002520513534546076,
      "perplexity_per_byte_labeled": 19.356298626901555,
      "perplexity_per_byte_ablated": 19.40514797643109
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.754284469286601,
      "eval_nll_ablated": 2.758339802424113,
      "label_use_gap": 0.004055333137511941,
      "perplexity_per_byte_labeled": 15.709796038693622,
      "perplexity_per_byte_ablated": 15.77363384945936
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.638744020462036,
      "eval_nll_ablated": 2.6537392218907674,
      "label_use_gap": 0.014995201428731342,
      "perplexity_per_byte_labeled": 13.995614358921506,
      "perplexity_per_byte_ablated": 14.207062809297595
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5766279776891072,
      "eval_nll_ablated": 2.5874828974405926,
      "label_use_gap": 0.010854919751485337,
      "perplexity_per_byte_labeled": 13.1527120603388,
      "perplexity_per_byte_ablated": 13.296261392987839
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9627971490224203,
      "eval_nll_ablated": 2.9652059078216553,
      "label_use_gap": 0.002408758799234967,
      "perplexity_per_byte_labeled": 19.352026622849074,
      "perplexity_per_byte_ablated": 19.398697173744285
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7537235657374066,
      "eval_nll_ablated": 2.7570571660995484,
      "label_use_gap": 0.0033336003621418087,
      "perplexity_per_byte_labeled": 15.700986829127814,
      "perplexity_per_byte_ablated": 15.753414983211742
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.638934826850891,
      "eval_nll_ablated": 2.651536560058594,
      "label_use_gap": 0.012601733207702637,
      "perplexity_per_byte_labeled": 13.998285066343058,
      "perplexity_per_byte_ablated": 14.175803893349089
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5772054115931193,
      "eval_nll_ablated": 2.585075569152832,
      "label_use_gap": 0.007870157559712876,
      "perplexity_per_byte_labeled": 13.160309075388088,
      "perplexity_per_byte_ablated": 13.264291423360605
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.963110987345378,
      "eval_nll_ablated": 2.965430466334025,
      "label_use_gap": 0.002319478988647372,
      "perplexity_per_byte_labeled": 19.35810098356395,
      "perplexity_per_byte_ablated": 19.403053805464598
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.754293513298035,
      "eval_nll_ablated": 2.757881840070089,
      "label_use_gap": 0.003588326772054007,
      "perplexity_per_byte_labeled": 15.709938118911108,
      "perplexity_per_byte_ablated": 15.766411772815983
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6384325504302977,
      "eval_nll_ablated": 2.6506051937739055,
      "label_use_gap": 0.012172643343607792,
      "perplexity_per_byte_labeled": 13.991255823284787,
      "perplexity_per_byte_ablated": 14.16260717398816
    },
    {
      "train_per_category": 100,
      "seed": 15109,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.576337122917175,
      "eval_nll_ablated": 2.5833288033803306,
      "label_use_gap": 0.0069916804631553475,
      "perplexity_per_byte_labeled": 13.14888708755496,
      "perplexity_per_byte_ablated": 13.24114203726953
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9232924302419026,
      "eval_nll_ablated": 2.9232924302419026,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 18.602433960360898,
      "perplexity_per_byte_ablated": 18.602433960360898
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7612234592437743,
      "eval_nll_ablated": 2.7612234592437743,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.819185242006956,
      "perplexity_per_byte_ablated": 15.819185242006956
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.639673391977946,
      "eval_nll_ablated": 2.639673391977946,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 14.008627530353118,
      "perplexity_per_byte_ablated": 14.008627530353118
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.571633044878642,
      "eval_nll_ablated": 2.571633044878642,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.087178950313403,
      "perplexity_per_byte_ablated": 13.087178950313403
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9337264855702716,
      "eval_nll_ablated": 2.9348182042439777,
      "label_use_gap": 0.0010917186737060547,
      "perplexity_per_byte_labeled": 18.797548935599707,
      "perplexity_per_byte_ablated": 18.818081776796348
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.761603418986003,
      "eval_nll_ablated": 2.7747339248657226,
      "label_use_gap": 0.013130505879719756,
      "perplexity_per_byte_labeled": 15.8251970376016,
      "perplexity_per_byte_ablated": 16.034360083508957
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6403689940770465,
      "eval_nll_ablated": 2.6550265312194825,
      "label_use_gap": 0.014657537142436006,
      "perplexity_per_byte_labeled": 14.018375350977946,
      "perplexity_per_byte_ablated": 14.22536347056436
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5744348605473837,
      "eval_nll_ablated": 2.586338249842326,
      "label_use_gap": 0.01190338929494228,
      "perplexity_per_byte_labeled": 13.12389822966108,
      "perplexity_per_byte_ablated": 13.281050566497228
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.933834425608317,
      "eval_nll_ablated": 2.9351104895273843,
      "label_use_gap": 0.0012760639190672052,
      "perplexity_per_byte_labeled": 18.79957805325653,
      "perplexity_per_byte_ablated": 18.823582829060793
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.76100119749705,
      "eval_nll_ablated": 2.776218628883362,
      "label_use_gap": 0.01521743138631182,
      "perplexity_per_byte_labeled": 15.815669632970488,
      "perplexity_per_byte_ablated": 16.058184043733597
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6405956983566283,
      "eval_nll_ablated": 2.656256413459778,
      "label_use_gap": 0.01566071510314959,
      "perplexity_per_byte_labeled": 14.021553736926032,
      "perplexity_per_byte_ablated": 14.242869755586394
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5736995379130048,
      "eval_nll_ablated": 2.5893429358800253,
      "label_use_gap": 0.015643397967020523,
      "perplexity_per_byte_labeled": 13.114251477413829,
      "perplexity_per_byte_ablated": 13.321015965370595
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9336079120635987,
      "eval_nll_ablated": 2.9347853501637777,
      "label_use_gap": 0.0011774381001790069,
      "perplexity_per_byte_labeled": 18.795320176444065,
      "perplexity_per_byte_ablated": 18.817463536184363
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7618539492289225,
      "eval_nll_ablated": 2.7726779619852704,
      "label_use_gap": 0.010824012756347923,
      "perplexity_per_byte_labeled": 15.829162224738589,
      "perplexity_per_byte_ablated": 16.00142789963958
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6403443177541095,
      "eval_nll_ablated": 2.65224822362264,
      "label_use_gap": 0.011903905868530629,
      "perplexity_per_byte_labeled": 14.018029433288739,
      "perplexity_per_byte_ablated": 14.185895887097233
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5731788953145345,
      "eval_nll_ablated": 2.5853219906489056,
      "label_use_gap": 0.012143095334371079,
      "perplexity_per_byte_labeled": 13.107425416570857,
      "perplexity_per_byte_ablated": 13.267560432658042
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.933584149678548,
      "eval_nll_ablated": 2.9351160208384197,
      "label_use_gap": 0.001531871159871745,
      "perplexity_per_byte_labeled": 18.794873560115235,
      "perplexity_per_byte_ablated": 18.82368694844018
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7611912886301675,
      "eval_nll_ablated": 2.7750792344411215,
      "label_use_gap": 0.013887945810953983,
      "perplexity_per_byte_labeled": 15.818676337296893,
      "perplexity_per_byte_ablated": 16.03989785764938
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.640859818458557,
      "eval_nll_ablated": 2.6532232920328775,
      "label_use_gap": 0.012363473574320327,
      "perplexity_per_byte_labeled": 14.025257600239076,
      "perplexity_per_byte_ablated": 14.199734851919914
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5733882109324138,
      "eval_nll_ablated": 2.5878486076990765,
      "label_use_gap": 0.014460396766662775,
      "perplexity_per_byte_labeled": 13.110169292578766,
      "perplexity_per_byte_ablated": 13.301124861434685
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.9336480776468914,
      "eval_nll_ablated": 2.9349647045135496,
      "label_use_gap": 0.0013166268666582326,
      "perplexity_per_byte_labeled": 18.79607511660333,
      "perplexity_per_byte_ablated": 18.820838832799268
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7620211044947305,
      "eval_nll_ablated": 2.7739606221516926,
      "label_use_gap": 0.011939517656962106,
      "perplexity_per_byte_labeled": 15.83180837371049,
      "perplexity_per_byte_ablated": 16.0219654623534
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6404433568318684,
      "eval_nll_ablated": 2.6528536081314087,
      "label_use_gap": 0.01241025129954032,
      "perplexity_per_byte_labeled": 14.019417834747673,
      "perplexity_per_byte_ablated": 14.194486408732283
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.573660643895467,
      "eval_nll_ablated": 2.5857879877090455,
      "label_use_gap": 0.012127343813578495,
      "perplexity_per_byte_labeled": 13.113741421405999,
      "perplexity_per_byte_ablated": 13.27374451758568
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.933789936701457,
      "eval_nll_ablated": 2.9349790891011556,
      "label_use_gap": 0.0011891523996987452,
      "perplexity_per_byte_labeled": 18.798741699183882,
      "perplexity_per_byte_ablated": 18.821109564751456
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7619502464930217,
      "eval_nll_ablated": 2.775005753835042,
      "label_use_gap": 0.013055507342020434,
      "perplexity_per_byte_labeled": 15.830686603149372,
      "perplexity_per_byte_ablated": 16.038719279535197
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6410929203033446,
      "eval_nll_ablated": 2.6552189111709597,
      "label_use_gap": 0.014125990867615013,
      "perplexity_per_byte_labeled": 14.028527294730415,
      "perplexity_per_byte_ablated": 14.22810040855613
    },
    {
      "train_per_category": 100,
      "seed": 16111,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.573819955190023,
      "eval_nll_ablated": 2.5881953636805215,
      "label_use_gap": 0.014375408490498565,
      "perplexity_per_byte_labeled": 13.115830754951112,
      "perplexity_per_byte_ablated": 13.305737905794475
    }
  ]
}