{
  "experiment": "H1 real-text conditional byte-model pilot v14 (v13 larger model, v12 categories)",
  "completed_utc": "2026-11-19T02:46:04.804243+00:00",
  "smoke_test_not_protocol": false,
  "protocol": {
    "dataset": "SetFit/20_newsgroups",
    "revision": "f1b91292074e7cfb69be58b642d583ec262f30ed",
    "dataset_split": "train only; official test unused",
    "selected_data_sha256": "9cbd6e8cf8d80bad34613d7930aa54da65872f8719dc62fb92bebea8f8071858",
    "categories": [
      "sci.crypt",
      "sci.electronics",
      "sci.med",
      "sci.space",
      "talk.politics.guns",
      "talk.politics.mideast",
      "talk.politics.misc",
      "soc.religion.christian"
    ],
    "sizes_train_per_category": [
      100
    ],
    "subsets_nested": true,
    "validation_per_category": 80,
    "validation_identical_across_sizes": true,
    "validation_messages": 640,
    "bytes_per_example": 96,
    "seed_values": [
      18163,
      19169,
      20173,
      21179,
      22189
    ],
    "steps": [
      100,
      200,
      400,
      800
    ],
    "total_updates": 800,
    "batch_size": 24,
    "conditions": [
      "none",
      "hierarchy",
      "scrambled_1",
      "scrambled_2",
      "scrambled_3",
      "scrambled_4",
      "scrambled_5"
    ],
    "permutation_seeds_selected": [
      12012,
      23023,
      34034,
      45045,
      56056
    ],
    "permutation_seeds_rejected": [],
    "permutation_seeds_primary": [
      12012,
      23023,
      34034,
      45045,
      56056
    ],
    "permutation_seeds_fallback": [
      67067,
      78078,
      89089,
      91919
    ],
    "scrambled_permutations": {
      "scrambled_1": [
        4,
        3,
        5,
        6,
        7,
        0,
        1,
        2
      ],
      "scrambled_2": [
        6,
        2,
        7,
        0,
        5,
        3,
        4,
        1
      ],
      "scrambled_3": [
        0,
        6,
        3,
        2,
        7,
        5,
        1,
        4
      ],
      "scrambled_4": [
        0,
        5,
        2,
        4,
        1,
        3,
        6,
        7
      ],
      "scrambled_5": [
        2,
        4,
        3,
        0,
        5,
        6,
        1,
        7
      ]
    },
    "permutation_rejection_rule": "rejected if categories 0-3 all fall in the same code//4 group (sci/politics split preserved), same as v11/v12/v13",
    "same_batch_sequence_per_size_and_seed_across_conditions": true,
    "batch_generator_seed": "seed + 30000, per (size, seed)",
    "evaluations": "each checkpoint: (a) eval_nll_labeled with the condition label prefix; (b) eval_nll_ablated with dummy prefix [256, 256] for all conditions; label_use_gap = ablated - labeled",
    "metric": "validation NLL in nats per byte, lower is better",
    "model": "randomly initialized 4-layer causal transformer width 128, 4 heads, FF 256, dropout 0.2 in training only (eval mode at every evaluation)",
    "dropout": 0.2,
    "weight_decay": 0.1,
    "learning_rate": 0.001,
    "regularisation_chosen_before_run": true,
    "v9_reference": "v9 used dropout 0 and weight decay 0.01",
    "replication_of": "v13 (model, training, rule, permutation rejection) with the v12 categories (sci vs politics/society)",
    "category_groups": {
      "group_0": [
        "sci.crypt",
        "sci.electronics",
        "sci.med",
        "sci.space"
      ],
      "group_1": [
        "talk.politics.guns",
        "talk.politics.mideast",
        "talk.politics.misc",
        "soc.religion.christian"
      ]
    },
    "category_set_differs_from_v10_v11_v13": true,
    "model_size_differs_from_v10_v12": true,
    "n_layers": 4,
    "d_model": 128,
    "n_heads": 4,
    "ff": 256,
    "v12_model": "2 layers, width 64, 4 heads, FF 128",
    "reference_v12_n100_paired_diff": {
      "200": -0.0001591691970824982,
      "400": -0.0001796048482258783
    },
    "reference_v12_n100_provenance": "v12.json replication_cells 100_200 and 100_400 mean_paired_diff; v12 = same categories, tiny model, seeds 12101..16111; unpaired with v14 (other model and seeds), indicative only",
    "preregistered_cells": [
      {
        "train_per_category": 100,
        "step": 200
      },
      {
        "train_per_category": 100,
        "step": 400
      }
    ],
    "new_seeds_and_permutations_vs_v10_v11_v12_v13": true,
    "v10_seeds_not_reused": [
      2027,
      3037,
      4049,
      5051,
      6067
    ],
    "v11_seeds_not_reused": [
      7079,
      8081,
      9091,
      10093,
      11099
    ],
    "v12_seeds_not_reused": [
      12101,
      13103,
      14107,
      15109,
      16111
    ],
    "v13_seeds_not_reused": [
      13121,
      14129,
      15137,
      16141,
      17149
    ],
    "primary_checkpoint": 800,
    "checkpoint_selection": "none; all checkpoints are reported and the primary criterion is fixed in advance at 800 updates",
    "limitations": "one larger model only (4 layers, width 128), not a scaling law; a single regularisation setting (dropout 0.2, weight decay 0.1) not tuned, which may under- or over-regularise the 4-layer model; the sci vs politics/society split may be more or less semantically clean than comp vs rec; five seeds, five permutations, two cells that share seeds across checkpoints and are not independent; comparison with v12 (and v13) is unpaired (other model or other categories, other seeds) and purely indicative, so v14 alone cannot separate model size from categories; no significance test is run or claimed; no data-savings percentage is estimated; labels present at labeled evaluation; cells chosen AFTER seeing v10; one historical text dataset (20 Newsgroups); not a pretrained-LLM or general data-savings result"
  },
  "environment": {
    "device": "cuda",
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition MIG 2g.48gb",
    "torch": "2.13.0+cu130",
    "cuda": "13.0",
    "python": "3.12.12",
    "code_sha256": "562a2e80c69d79cc1d32ef4582b767622dd4a695a6630f72b4e11c87fdc65321"
  },
  "summary": {
    "100": {
      "100": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.715721936225891,
          "hierarchy": 2.7166292651494346,
          "scrambled_1": 2.716794722874959,
          "scrambled_2": 2.71569987932841,
          "scrambled_3": 2.7163586584726973,
          "scrambled_4": 2.7167037804921472,
          "scrambled_5": 2.715952518781026,
          "scrambled_mean": 2.7163019119898477
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": 0.0003273531595866075,
          "min": 0.00010981082916217133,
          "max": 0.0005795621871951262,
          "per_seed": {
            "18163": 0.0005584081013996212,
            "19169": 0.0002345975240074516,
            "20173": 0.00015438715616866716,
            "21179": 0.0005795621871951262,
            "22189": 0.00010981082916217133
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 3.6,
          "per_seed": {
            "18163": 3,
            "19169": 4,
            "20173": 4,
            "21179": 4,
            "22189": 3
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 4.6,
          "per_seed": {
            "18163": 4,
            "19169": 5,
            "20173": 5,
            "21179": 5,
            "22189": 4
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.002411206563313861,
          "per_seed": {
            "18163": 0.0030341545740766307,
            "19169": 0.0008231163024903232,
            "20173": 0.0011707544326782227,
            "21179": 0.005520677566528409,
            "22189": 0.0015073299407957208
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 0
      },
      "200": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.6121490939458214,
          "hierarchy": 2.613663549423218,
          "scrambled_1": 2.613977627754211,
          "scrambled_2": 2.6137268861134846,
          "scrambled_3": 2.6140927616755167,
          "scrambled_4": 2.6140133333206177,
          "scrambled_5": 2.6142818212509153,
          "scrambled_mean": 2.614018486022949
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.000354936599731559,
          "min": -0.0010463682810466501,
          "max": 0.0003630892435708688,
          "per_seed": {
            "18163": -0.000837411880493022,
            "19169": 0.00025758743286097285,
            "20173": -0.0010463682810466501,
            "21179": 0.0003630892435708688,
            "22189": -0.0005115795135499646
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.8,
          "per_seed": {
            "18163": 1,
            "19169": 3,
            "20173": 0,
            "21179": 3,
            "22189": 2
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.8,
          "per_seed": {
            "18163": 2,
            "19169": 4,
            "20173": 1,
            "21179": 4,
            "22189": 3
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0017822949091593544,
          "per_seed": {
            "18163": 0.0017082373301189335,
            "19169": 0.0009778420130412968,
            "20173": 0.0019193251927691612,
            "21179": 0.0028014024098714785,
            "22189": 0.0015046675999959014
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 3
      },
      "400": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.4946047004063923,
          "hierarchy": 2.498068935076396,
          "scrambled_1": 2.497592412630717,
          "scrambled_2": 2.4975127172470093,
          "scrambled_3": 2.498479633331299,
          "scrambled_4": 2.4976735575993856,
          "scrambled_5": 2.4978400309880575,
          "scrambled_mean": 2.4978196703592936
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": 0.0002492647171019158,
          "min": -0.0019284168879192265,
          "max": 0.002169473965962787,
          "per_seed": {
            "18163": -0.0019284168879192265,
            "19169": 0.002169473965962787,
            "20173": -0.001169711748759017,
            "21179": 0.0017712211608884587,
            "22189": 0.00040375709533657655
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 2.6,
          "per_seed": {
            "18163": 0,
            "19169": 5,
            "20173": 1,
            "21179": 4,
            "22189": 3
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 3.6,
          "per_seed": {
            "18163": 1,
            "19169": 6,
            "20173": 2,
            "21179": 5,
            "22189": 4
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0050313059488932945,
          "per_seed": {
            "18163": 0.0012998342514038974,
            "19169": 0.0024779637654623876,
            "20173": 0.006988008817037095,
            "21179": 0.007019639015197754,
            "22189": 0.007371083895365338
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 2
      },
      "800": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.3672173309326174,
          "hierarchy": 2.375312755902608,
          "scrambled_1": 2.376982545852661,
          "scrambled_2": 2.3774563964207966,
          "scrambled_3": 2.3752968279520674,
          "scrambled_4": 2.377295338312785,
          "scrambled_5": 2.3772696749369304,
          "scrambled_mean": 2.376860156695048
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.001547400792439735,
          "min": -0.013880615234374893,
          "max": 0.0059847561518351355,
          "per_seed": {
            "18163": -0.0011729717254640448,
            "19169": -0.0013683048884072235,
            "20173": -0.013880615234374893,
            "21179": 0.002700131734212352,
            "22189": 0.0059847561518351355
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 2.0,
          "per_seed": {
            "18163": 1,
            "19169": 1,
            "20173": 0,
            "21179": 4,
            "22189": 4
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 3.0,
          "per_seed": {
            "18163": 2,
            "19169": 2,
            "20173": 1,
            "21179": 5,
            "22189": 5
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.007838479677836041,
          "per_seed": {
            "18163": 0.005353983243306271,
            "19169": 0.0037293275197347597,
            "20173": 0.00884049733479797,
            "21179": 0.006764634450276752,
            "22189": 0.014503955841064453
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 3
      }
    }
  },
  "replication_cells": {
    "cells": {
      "100_200": {
        "train_per_category": 100,
        "step": 200,
        "mean_paired_diff": -0.000354936599731559,
        "seeds_hierarchy_better": 3,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.0017822949091593544,
        "sign_negative": true,
        "wins_at_least_4_of_5": false,
        "material": false
      },
      "100_400": {
        "train_per_category": 100,
        "step": 400,
        "mean_paired_diff": 0.0002492647171019158,
        "seeds_hierarchy_better": 2,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.0050313059488932945,
        "sign_negative": false,
        "wins_at_least_4_of_5": false,
        "material": false
      }
    },
    "missing_cells": [],
    "n_cells_sign_negative": 1,
    "n_cells_wins_at_least_4_of_5": 0,
    "n_cells_material": 0
  },
  "reference_v12_n100": {
    "200": -0.0001591691970824982,
    "400": -0.0001796048482258783
  },
  "p5_vs_v12_indicative": {
    "v14_mean_paired_diff_100_400": 0.0002492647171019158,
    "v12_reference_100_400": -0.0001796048482258783,
    "v14_more_negative_than_v12": false,
    "unpaired_indicative_only": true
  },
  "summary_label_use": {
    "100": {
      "100": {
        "none": 0.0,
        "hierarchy": 0.0048465808232624145,
        "scrambled_1": 0.005478555361429827,
        "scrambled_2": 0.005382421811421701,
        "scrambled_3": 0.005349289576212524,
        "scrambled_4": 0.004583826065063334,
        "scrambled_5": 0.00526559193929046,
        "scrambled_mean": 0.005211936950683569
      },
      "200": {
        "none": 0.0,
        "hierarchy": 0.0032872327168783854,
        "scrambled_1": 0.0034178829193117545,
        "scrambled_2": 0.004074498812357686,
        "scrambled_3": 0.003825654983520632,
        "scrambled_4": 0.0031200997034708244,
        "scrambled_5": 0.0032624133427937353,
        "scrambled_mean": 0.0035401099522909263
      },
      "400": {
        "none": 0.0,
        "hierarchy": 0.009836506843566984,
        "scrambled_1": 0.00966027418772386,
        "scrambled_2": 0.008101784388224332,
        "scrambled_3": 0.009555492401123012,
        "scrambled_4": 0.008432927131652867,
        "scrambled_5": 0.007981885274251432,
        "scrambled_mean": 0.008746472676595101
      },
      "800": {
        "none": 0.0,
        "hierarchy": 0.017911281585693307,
        "scrambled_1": 0.017247263590494732,
        "scrambled_2": 0.01921516259511318,
        "scrambled_3": 0.020550376574198382,
        "scrambled_4": 0.017317598660786793,
        "scrambled_5": 0.01880188941955572,
        "scrambled_mean": 0.01862645816802976
      }
    }
  },
  "elapsed_seconds": 190.084664650989,
  "runs": [
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.717322071393331,
      "eval_nll_ablated": 2.717322071393331,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.13972480853522,
      "perplexity_per_byte_ablated": 15.13972480853522
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6089744249979656,
      "eval_nll_ablated": 2.6089744249979656,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.585111154550246,
      "perplexity_per_byte_ablated": 13.585111154550246
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.4941182374954223,
      "eval_nll_ablated": 2.4941182374954223,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.111049739452417,
      "perplexity_per_byte_ablated": 12.111049739452417
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3712186018625894,
      "eval_nll_ablated": 2.3712186018625894,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.710436092263613,
      "perplexity_per_byte_ablated": 10.710436092263613
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7165902058283486,
      "eval_nll_ablated": 2.716203141212463,
      "label_use_gap": -0.00038706461588544627,
      "perplexity_per_byte_labeled": 15.128648618919822,
      "perplexity_per_byte_ablated": 15.122793987486702
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.608170262972514,
      "eval_nll_ablated": 2.6104482332865397,
      "label_use_gap": 0.002277970314025879,
      "perplexity_per_byte_labeled": 13.574190915457498,
      "perplexity_per_byte_ablated": 13.605147765405789
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.4877294301986694,
      "eval_nll_ablated": 2.4907227675120036,
      "label_use_gap": 0.0029933373133341767,
      "perplexity_per_byte_labeled": 12.033921218479605,
      "perplexity_per_byte_ablated": 12.069996770099372
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3858723640441895,
      "eval_nll_ablated": 2.3912058591842653,
      "label_use_gap": 0.005333495140075861,
      "perplexity_per_byte_labeled": 10.868539853921428,
      "perplexity_per_byte_ablated": 10.926662017771903
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7152164141337076,
      "eval_nll_ablated": 2.714904308319092,
      "label_use_gap": -0.0003121058146158262,
      "perplexity_per_byte_labeled": 15.107879276737409,
      "perplexity_per_byte_ablated": 15.103164755521638
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.608996041615804,
      "eval_nll_ablated": 2.6113959391911825,
      "label_use_gap": 0.002399897575378507,
      "perplexity_per_byte_labeled": 13.585404821880402,
      "perplexity_per_byte_ablated": 13.618047555914927
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.4889819860458373,
      "eval_nll_ablated": 2.4899876356124877,
      "label_use_gap": 0.0010056495666503906,
      "perplexity_per_byte_labeled": 12.049003820795111,
      "perplexity_per_byte_ablated": 12.06112699107482
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3860511620839437,
      "eval_nll_ablated": 2.3934505621592206,
      "label_use_gap": 0.007399400075276841,
      "perplexity_per_byte_labeled": 10.870483301279412,
      "perplexity_per_byte_ablated": 10.951216677157735
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.714630397160848,
      "eval_nll_ablated": 2.7149381240208945,
      "label_use_gap": 0.00030772686004665317,
      "perplexity_per_byte_labeled": 15.099028396693583,
      "perplexity_per_byte_ablated": 15.103675488272632
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.609321117401123,
      "eval_nll_ablated": 2.6116975069046022,
      "label_use_gap": 0.0023763895034791815,
      "perplexity_per_byte_labeled": 13.589821825913383,
      "perplexity_per_byte_ablated": 13.62215493867349
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.490281820297241,
      "eval_nll_ablated": 2.492267950375875,
      "label_use_gap": 0.0019861300786336855,
      "perplexity_per_byte_labeled": 12.064675711880524,
      "perplexity_per_byte_ablated": 12.08866153880224
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3870531797409056,
      "eval_nll_ablated": 2.3932629346847536,
      "label_use_gap": 0.0062097549438480115,
      "perplexity_per_byte_labeled": 10.88138117650688,
      "perplexity_per_byte_ablated": 10.949162120781898
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7155415852864584,
      "eval_nll_ablated": 2.715603311856588,
      "label_use_gap": 6.172657012948335e-05,
      "perplexity_per_byte_labeled": 15.112792722069493,
      "perplexity_per_byte_ablated": 15.113725611721046
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.608736745516459,
      "eval_nll_ablated": 2.611479059855143,
      "label_use_gap": 0.002742314338683993,
      "perplexity_per_byte_labeled": 13.581882636066323,
      "perplexity_per_byte_ablated": 13.619179544114775
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.490084250768026,
      "eval_nll_ablated": 2.4915796836217243,
      "label_use_gap": 0.001495432853698464,
      "perplexity_per_byte_labeled": 12.062292335029065,
      "perplexity_per_byte_ablated": 12.080344177572847
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3860262711842855,
      "eval_nll_ablated": 2.392650032043457,
      "label_use_gap": 0.00662376085917149,
      "perplexity_per_byte_labeled": 10.870212728537737,
      "perplexity_per_byte_ablated": 10.942453406502446
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7176645517349245,
      "eval_nll_ablated": 2.7172972281773884,
      "label_use_gap": -0.00036732355753610335,
      "perplexity_per_byte_labeled": 15.144910754650889,
      "perplexity_per_byte_ablated": 15.13934869375446
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.608138116200765,
      "eval_nll_ablated": 2.609571623802185,
      "label_use_gap": 0.0014335076014200254,
      "perplexity_per_byte_labeled": 13.573754556054272,
      "perplexity_per_byte_ablated": 13.593226589709822
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.489095346132914,
      "eval_nll_ablated": 2.490655946731567,
      "label_use_gap": 0.0015606005986530391,
      "perplexity_per_byte_labeled": 12.05036977433827,
      "perplexity_per_byte_ablated": 12.0691902704411
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3853710412979128,
      "eval_nll_ablated": 2.3876743793487547,
      "label_use_gap": 0.00230333805084193,
      "perplexity_per_byte_labeled": 10.863092573210793,
      "perplexity_per_byte_ablated": 10.88814278616378
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7171060403188068,
      "eval_nll_ablated": 2.7170248587926227,
      "label_use_gap": -8.118152618408203e-05,
      "perplexity_per_byte_labeled": 15.13645451077252,
      "perplexity_per_byte_ablated": 15.135225760170918
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.609846353530884,
      "eval_nll_ablated": 2.6124497413635255,
      "label_use_gap": 0.0026033878326416904,
      "perplexity_per_byte_labeled": 13.596961566194,
      "perplexity_per_byte_ablated": 13.632405848083419
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.4898458321889243,
      "eval_nll_ablated": 2.492921988169352,
      "label_use_gap": 0.003076155980427764,
      "perplexity_per_byte_labeled": 12.059416803233564,
      "perplexity_per_byte_ablated": 12.096570566440729
    },
    {
      "train_per_category": 100,
      "seed": 18163,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.390725024541219,
      "eval_nll_ablated": 2.3941726287206015,
      "label_use_gap": 0.003447604179382502,
      "perplexity_per_byte_labeled": 10.921409363071277,
      "perplexity_per_byte_ablated": 10.959127040087335
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.6970412731170654,
      "eval_nll_ablated": 2.6970412731170654,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 14.835771727489664,
      "perplexity_per_byte_ablated": 14.835771727489664
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6075263897577923,
      "eval_nll_ablated": 2.6075263897577923,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.56545367065632,
      "perplexity_per_byte_ablated": 13.56545367065632
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.4698928594589233,
      "eval_nll_ablated": 2.4698928594589233,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 11.821180256146901,
      "perplexity_per_byte_ablated": 11.821180256146901
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3566667477289838,
      "eval_nll_ablated": 2.3566667477289838,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.555707909893156,
      "perplexity_per_byte_ablated": 10.555707909893156
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.6997162818908693,
      "eval_nll_ablated": 2.6989229281743365,
      "label_use_gap": -0.0007933537165327742,
      "perplexity_per_byte_labeled": 14.875510674346033,
      "perplexity_per_byte_ablated": 14.863713812827937
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6114448706309,
      "eval_nll_ablated": 2.6098820130030314,
      "label_use_gap": -0.0015628576278685635,
      "perplexity_per_byte_labeled": 13.618713922890992,
      "perplexity_per_byte_ablated": 13.5974464353113
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.478604777654012,
      "eval_nll_ablated": 2.490692059199015,
      "label_use_gap": 0.012087281545003048,
      "perplexity_per_byte_labeled": 11.92461531619413,
      "perplexity_per_byte_ablated": 12.069626126551736
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3529392083485923,
      "eval_nll_ablated": 2.3697734514872235,
      "label_use_gap": 0.016834243138631244,
      "perplexity_per_byte_labeled": 10.516434335343034,
      "perplexity_per_byte_ablated": 10.694969080295476
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.6991646687189736,
      "eval_nll_ablated": 2.699456485112508,
      "label_use_gap": 0.00029181639353437205,
      "perplexity_per_byte_labeled": 14.867307409441882,
      "perplexity_per_byte_ablated": 14.87164656655947
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6107213656107584,
      "eval_nll_ablated": 2.6091554005940756,
      "label_use_gap": -0.0015659650166828243,
      "perplexity_per_byte_labeled": 13.608864278563125,
      "perplexity_per_byte_ablated": 13.587569950623992
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.475101868311564,
      "eval_nll_ablated": 2.4866740226745607,
      "label_use_gap": 0.011572154362996745,
      "perplexity_per_byte_labeled": 11.882917544192747,
      "perplexity_per_byte_ablated": 12.021227227326282
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3557615280151367,
      "eval_nll_ablated": 2.3659802357355755,
      "label_use_gap": 0.010218707720438758,
      "perplexity_per_byte_labeled": 10.546156998488339,
      "perplexity_per_byte_ablated": 10.65447760012781
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.6992528597513834,
      "eval_nll_ablated": 2.698948097229004,
      "label_use_gap": -0.00030476252237932044,
      "perplexity_per_byte_labeled": 14.86861863044959,
      "perplexity_per_byte_ablated": 14.864087923161437
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6106746355692545,
      "eval_nll_ablated": 2.6105114221572876,
      "label_use_gap": -0.00016321341196690042,
      "perplexity_per_byte_labeled": 13.608228350629153,
      "perplexity_per_byte_ablated": 13.606007486491558
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.4775798320770264,
      "eval_nll_ablated": 2.4892078399658204,
      "label_use_gap": 0.011628007888794034,
      "perplexity_per_byte_labeled": 11.912399495813482,
      "perplexity_per_byte_ablated": 12.051725442872867
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3552472591400146,
      "eval_nll_ablated": 2.371519184112549,
      "label_use_gap": 0.01627192497253427,
      "perplexity_per_byte_labeled": 10.540734832536959,
      "perplexity_per_byte_ablated": 10.713655943133041
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.699678325653076,
      "eval_nll_ablated": 2.6997035344441733,
      "label_use_gap": 2.5208791097242056e-05,
      "perplexity_per_byte_labeled": 14.874946066640842,
      "perplexity_per_byte_ablated": 14.875321050775247
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6112465778986613,
      "eval_nll_ablated": 2.610598659515381,
      "label_use_gap": -0.000647918383280377,
      "perplexity_per_byte_labeled": 13.616013698623918,
      "perplexity_per_byte_ablated": 13.607194490413669
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.4758668661117555,
      "eval_nll_ablated": 2.490761637687683,
      "label_use_gap": 0.014894771575927557,
      "perplexity_per_byte_labeled": 11.892011427930944,
      "perplexity_per_byte_ablated": 12.070465942112618
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.352032200495402,
      "eval_nll_ablated": 2.3688544829686484,
      "label_use_gap": 0.016822282473246464,
      "perplexity_per_byte_labeled": 10.50690017124779,
      "perplexity_per_byte_ablated": 10.685145254987999
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.699987785021464,
      "eval_nll_ablated": 2.699863338470459,
      "label_use_gap": -0.00012444655100507518,
      "perplexity_per_byte_labeled": 14.879549970379259,
      "perplexity_per_byte_ablated": 14.877698376919538
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.611641359329224,
      "eval_nll_ablated": 2.6107394297917685,
      "label_use_gap": -0.0009019295374552705,
      "perplexity_per_byte_labeled": 13.621390109174117,
      "perplexity_per_byte_ablated": 13.609110113771196
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.4760587930679323,
      "eval_nll_ablated": 2.4893980662027997,
      "label_use_gap": 0.01333927313486738,
      "perplexity_per_byte_labeled": 11.894294044527948,
      "perplexity_per_byte_ablated": 12.054018215318797
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.35502773920695,
      "eval_nll_ablated": 2.3694382429122927,
      "label_use_gap": 0.014410503705342581,
      "perplexity_per_byte_labeled": 10.538421185087227,
      "perplexity_per_byte_ablated": 10.691384635752977
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.6993247826894122,
      "eval_nll_ablated": 2.6991830269495645,
      "label_use_gap": -0.0001417557398477598,
      "perplexity_per_byte_labeled": 14.869688063643851,
      "perplexity_per_byte_ablated": 14.867580349404909
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.611652477582296,
      "eval_nll_ablated": 2.6096064249674478,
      "label_use_gap": -0.0020460526148480263,
      "perplexity_per_byte_labeled": 13.621541556078453,
      "perplexity_per_byte_ablated": 13.593699658066443
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.4775691588719684,
      "eval_nll_ablated": 2.4913313070933025,
      "label_use_gap": 0.013762148221334147,
      "perplexity_per_byte_labeled": 11.912272353009442,
      "perplexity_per_byte_ablated": 12.077344076216685
    },
    {
      "train_per_category": 100,
      "seed": 19169,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3534688393274945,
      "eval_nll_ablated": 2.375265630086263,
      "label_use_gap": 0.021796790758768392,
      "perplexity_per_byte_labeled": 10.522005639992157,
      "perplexity_per_byte_ablated": 10.753869357962184
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.717946728070577,
      "eval_nll_ablated": 2.717946728070577,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.149184893073421,
      "perplexity_per_byte_ablated": 15.149184893073421
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6145390272140503,
      "eval_nll_ablated": 2.6145390272140503,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.660917614974572,
      "perplexity_per_byte_ablated": 13.660917614974572
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5018314123153687,
      "eval_nll_ablated": 2.5018314123153687,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.2048255731282,
      "perplexity_per_byte_ablated": 12.2048255731282
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.361155605316162,
      "eval_nll_ablated": 2.361155605316162,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.603197486663754,
      "perplexity_per_byte_ablated": 10.603197486663754
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.719821230570475,
      "eval_nll_ablated": 2.7233969291051228,
      "label_use_gap": 0.0035756985346475645,
      "perplexity_per_byte_labeled": 15.177608709961941,
      "perplexity_per_byte_ablated": 15.23197640650499
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6160867691040037,
      "eval_nll_ablated": 2.617889149983724,
      "label_use_gap": 0.001802380879720289,
      "perplexity_per_byte_labeled": 13.682077560265267,
      "perplexity_per_byte_ablated": 13.706760112252692
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5069471756617228,
      "eval_nll_ablated": 2.514283029238383,
      "label_use_gap": 0.007335853576660334,
      "perplexity_per_byte_labeled": 12.267422551588941,
      "perplexity_per_byte_ablated": 12.357745459947239
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3653667688369753,
      "eval_nll_ablated": 2.3830937226613362,
      "label_use_gap": 0.017726953824360958,
      "perplexity_per_byte_labeled": 10.64794343525011,
      "perplexity_per_byte_ablated": 10.838381997649543
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.719535207748413,
      "eval_nll_ablated": 2.7253836075464886,
      "label_use_gap": 0.005848399798075388,
      "perplexity_per_byte_labeled": 15.17326818826029,
      "perplexity_per_byte_ablated": 15.26226752504429
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.618054175376892,
      "eval_nll_ablated": 2.620800320307414,
      "label_use_gap": 0.002746144930521943,
      "perplexity_per_byte_labeled": 13.709022262380248,
      "perplexity_per_byte_ablated": 13.74672096372582
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5123961051305135,
      "eval_nll_ablated": 2.519437630971273,
      "label_use_gap": 0.007041525840759455,
      "perplexity_per_byte_labeled": 12.334449318057949,
      "perplexity_per_byte_ablated": 12.421609170702352
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.382288400332133,
      "eval_nll_ablated": 2.396088473002116,
      "label_use_gap": 0.013800072669982733,
      "perplexity_per_byte_labeled": 10.829657120255236,
      "perplexity_per_byte_ablated": 10.980143146322721
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7197265148162844,
      "eval_nll_ablated": 2.7229183117548623,
      "label_use_gap": 0.003191796938577962,
      "perplexity_per_byte_labeled": 15.17617121938374,
      "perplexity_per_byte_ablated": 15.224687862668983
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.616655882199605,
      "eval_nll_ablated": 2.619935464859009,
      "label_use_gap": 0.003279582659403868,
      "perplexity_per_byte_labeled": 13.689866425942348,
      "perplexity_per_byte_ablated": 13.734837176822296
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.508032528559367,
      "eval_nll_ablated": 2.512747852007548,
      "label_use_gap": 0.004715323448181152,
      "perplexity_per_byte_labeled": 12.280744262272844,
      "perplexity_per_byte_ablated": 12.338788685220202
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3832016388575235,
      "eval_nll_ablated": 2.3951725006103515,
      "label_use_gap": 0.011970861752828021,
      "perplexity_per_byte_labeled": 10.839551697721209,
      "perplexity_per_byte_ablated": 10.970090243136827
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7204456408818563,
      "eval_nll_ablated": 2.7238220453262327,
      "label_use_gap": 0.0033764044443764796,
      "perplexity_per_byte_labeled": 15.187088724744028,
      "perplexity_per_byte_ablated": 15.238453143340362
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.617517884572347,
      "eval_nll_ablated": 2.6208529710769652,
      "label_use_gap": 0.0033350865046184452,
      "perplexity_per_byte_labeled": 13.70167221086031,
      "perplexity_per_byte_ablated": 13.74744475821737
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.507556613286336,
      "eval_nll_ablated": 2.5135448296864826,
      "label_use_gap": 0.005988216400146573,
      "perplexity_per_byte_labeled": 12.274901059059355,
      "perplexity_per_byte_ablated": 12.348626344063948
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3768729686737062,
      "eval_nll_ablated": 2.3924231926600137,
      "label_use_gap": 0.015550223986307454,
      "perplexity_per_byte_labeled": 10.77116836610283,
      "perplexity_per_byte_ablated": 10.939971508625094
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7193519671758017,
      "eval_nll_ablated": 2.7237457354863484,
      "label_use_gap": 0.0043937683105466085,
      "perplexity_per_byte_labeled": 15.170488084630755,
      "perplexity_per_byte_ablated": 15.237290343787997
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.616134850184123,
      "eval_nll_ablated": 2.6183974107106525,
      "label_use_gap": 0.0022625605265296223,
      "perplexity_per_byte_labeled": 13.68273542514794,
      "perplexity_per_byte_ablated": 13.71372849083775
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5071910937627155,
      "eval_nll_ablated": 2.5138071457544964,
      "label_use_gap": 0.006616051991780836,
      "perplexity_per_byte_labeled": 12.270415162963005,
      "perplexity_per_byte_ablated": 12.351866012061803
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3743611415227255,
      "eval_nll_ablated": 2.392558725674947,
      "label_use_gap": 0.018197584152221502,
      "perplexity_per_byte_labeled": 10.744147003656094,
      "perplexity_per_byte_ablated": 10.941454336430734
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.719274886449178,
      "eval_nll_ablated": 2.723790709177653,
      "label_use_gap": 0.004515822728475172,
      "perplexity_per_byte_labeled": 15.169318777452057,
      "perplexity_per_byte_ablated": 15.237975636390193
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.617302894592285,
      "eval_nll_ablated": 2.6217458804448444,
      "label_use_gap": 0.0044429858525592,
      "perplexity_per_byte_labeled": 13.698726805252578,
      "perplexity_per_byte_ablated": 13.759725462388344
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5054080963134764,
      "eval_nll_ablated": 2.5117279211680095,
      "label_use_gap": 0.006319824854533085,
      "perplexity_per_byte_labeled": 12.248556536754583,
      "perplexity_per_byte_ablated": 12.326210389703737
    },
    {
      "train_per_category": 100,
      "seed": 20173,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3795127709706625,
      "eval_nll_ablated": 2.3980523109436036,
      "label_use_gap": 0.01853953997294111,
      "perplexity_per_byte_labeled": 10.799639683887952,
      "perplexity_per_byte_ablated": 11.001727555240048
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.717942142486572,
      "eval_nll_ablated": 2.717942142486572,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.149115425372763,
      "perplexity_per_byte_ablated": 15.149115425372763
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6100985844930014,
      "eval_nll_ablated": 2.6100985844930014,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.600391573450548,
      "perplexity_per_byte_ablated": 13.600391573450548
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5058743794759115,
      "eval_nll_ablated": 2.5058743794759115,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.254269164179295,
      "perplexity_per_byte_ablated": 12.254269164179295
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3821040709813435,
      "eval_nll_ablated": 2.3821040709813435,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.827661080558986,
      "perplexity_per_byte_ablated": 10.827661080558986
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7181735674540204,
      "eval_nll_ablated": 2.7314626216888427,
      "label_use_gap": 0.013289054234822295,
      "perplexity_per_byte_labeled": 15.152621714622738,
      "perplexity_per_byte_ablated": 15.355329641546781
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6108487606048585,
      "eval_nll_ablated": 2.617670011520386,
      "label_use_gap": 0.006821250915527433,
      "perplexity_per_byte_labeled": 13.610598090184718,
      "perplexity_per_byte_ablated": 13.703756762991038
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.502414353688558,
      "eval_nll_ablated": 2.5203517198562624,
      "label_use_gap": 0.01793736616770447,
      "perplexity_per_byte_labeled": 12.211942345036194,
      "perplexity_per_byte_ablated": 12.432968816651053
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.38266548315684,
      "eval_nll_ablated": 2.4100915511449177,
      "label_use_gap": 0.027426067988077563,
      "perplexity_per_byte_labeled": 10.833741567991895,
      "perplexity_per_byte_ablated": 11.1349805186173
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.721212967236837,
      "eval_nll_ablated": 2.7361450672149656,
      "label_use_gap": 0.014932099978128832,
      "perplexity_per_byte_labeled": 15.198746650322716,
      "perplexity_per_byte_ablated": 15.42739873425137
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.610357681910197,
      "eval_nll_ablated": 2.617486071586609,
      "label_use_gap": 0.007128389676412272,
      "perplexity_per_byte_labeled": 13.603915856326655,
      "perplexity_per_byte_ablated": 13.70123632669111
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5003188451131186,
      "eval_nll_ablated": 2.5151079893112183,
      "label_use_gap": 0.014789144198099624,
      "perplexity_per_byte_labeled": 12.186378908683867,
      "perplexity_per_byte_ablated": 12.367944312786737
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3843321561813355,
      "eval_nll_ablated": 2.404370824495951,
      "label_use_gap": 0.02003866831461565,
      "perplexity_per_byte_labeled": 10.851812928158658,
      "perplexity_per_byte_ablated": 11.071462197533513
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.716888682047526,
      "eval_nll_ablated": 2.7309852600097657,
      "label_use_gap": 0.014096577962239731,
      "perplexity_per_byte_labeled": 15.13316483471884,
      "perplexity_per_byte_ablated": 15.348001344869477
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6088433186213176,
      "eval_nll_ablated": 2.616145356496175,
      "label_use_gap": 0.007302037874857614,
      "perplexity_per_byte_labeled": 13.583330176601747,
      "perplexity_per_byte_ablated": 13.682879180991215
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.4968395709991453,
      "eval_nll_ablated": 2.5073108116785687,
      "label_use_gap": 0.0104712406794234,
      "perplexity_per_byte_labeled": 12.144052830715175,
      "perplexity_per_byte_ablated": 12.271884239428601
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3792630751927693,
      "eval_nll_ablated": 2.4022161960601807,
      "label_use_gap": 0.022953120867411414,
      "perplexity_per_byte_labeled": 10.796943396095964,
      "perplexity_per_byte_ablated": 11.0476329910189
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.717169761657715,
      "eval_nll_ablated": 2.7317216714223225,
      "label_use_gap": 0.014551909764607629,
      "perplexity_per_byte_labeled": 15.13741905665102,
      "perplexity_per_byte_ablated": 15.359307950865656
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6106100638707477,
      "eval_nll_ablated": 2.617241136233012,
      "label_use_gap": 0.006631072362264234,
      "perplexity_per_byte_labeled": 13.607349672580057,
      "perplexity_per_byte_ablated": 13.697880820484539
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.50035609404246,
      "eval_nll_ablated": 2.514470672607422,
      "label_use_gap": 0.0141145785649619,
      "perplexity_per_byte_labeled": 12.186832846705066,
      "perplexity_per_byte_ablated": 12.360064526510996
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.377567521731059,
      "eval_nll_ablated": 2.4111908197402956,
      "label_use_gap": 0.0336232980092368,
      "perplexity_per_byte_labeled": 10.77865211265144,
      "perplexity_per_byte_ablated": 11.147227583185023
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7170063257217407,
      "eval_nll_ablated": 2.7282697280248005,
      "label_use_gap": 0.011263402303059777,
      "perplexity_per_byte_labeled": 15.134945260558359,
      "perplexity_per_byte_ablated": 15.306379894079408
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.611644721031189,
      "eval_nll_ablated": 2.6187993049621583,
      "label_use_gap": 0.007154583930969238,
      "perplexity_per_byte_labeled": 13.621435900304984,
      "perplexity_per_byte_ablated": 13.719241067147095
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.503859210014343,
      "eval_nll_ablated": 2.516275326410929,
      "label_use_gap": 0.01241611639658613,
      "perplexity_per_byte_labeled": 12.229599600210173,
      "perplexity_per_byte_ablated": 12.382390303059971
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.380733331044515,
      "eval_nll_ablated": 2.4087859869003294,
      "label_use_gap": 0.02805265585581429,
      "perplexity_per_byte_labeled": 10.812829340644852,
      "perplexity_per_byte_ablated": 11.12045257183551
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7156922896703084,
      "eval_nll_ablated": 2.7291335582733156,
      "label_use_gap": 0.013441268603007206,
      "perplexity_per_byte_labeled": 15.115070457812994,
      "perplexity_per_byte_ablated": 15.319607720502917
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6109725713729857,
      "eval_nll_ablated": 2.6166765292485556,
      "label_use_gap": 0.005703957875569987,
      "perplexity_per_byte_labeled": 13.612283333112392,
      "perplexity_per_byte_ablated": 13.69014908420259
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.501841942469279,
      "eval_nll_ablated": 2.5105284293492636,
      "label_use_gap": 0.008686486879984745,
      "perplexity_per_byte_labeled": 12.204954092496596,
      "perplexity_per_byte_ablated": 12.311434064997103
    },
    {
      "train_per_category": 100,
      "seed": 21179,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.37793067296346,
      "eval_nll_ablated": 2.4001341660817466,
      "label_use_gap": 0.0222034931182864,
      "perplexity_per_byte_labeled": 10.782567104273765,
      "perplexity_per_byte_ablated": 11.02465541624098
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.72835746606191,
      "eval_nll_ablated": 2.72835746606191,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.307722904722262,
      "perplexity_per_byte_ablated": 15.307722904722262
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6196070432662966,
      "eval_nll_ablated": 2.6196070432662966,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.730327100364823,
      "perplexity_per_byte_ablated": 13.730327100364823
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5013066132863364,
      "eval_nll_ablated": 2.5013066132863364,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.198422172913988,
      "perplexity_per_byte_ablated": 12.198422172913988
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.364941628774007,
      "eval_nll_ablated": 2.364941628774007,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.643417530047547,
      "perplexity_per_byte_ablated": 10.643417530047547
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7288450400034585,
      "eval_nll_ablated": 2.737393609682719,
      "label_use_gap": 0.008548569679260432,
      "perplexity_per_byte_labeled": 15.31518837135065,
      "perplexity_per_byte_ablated": 15.446672526308236
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6217670838038125,
      "eval_nll_ablated": 2.6288645029067994,
      "label_use_gap": 0.0070974191029868905,
      "perplexity_per_byte_labeled": 13.760017217879714,
      "perplexity_per_byte_ablated": 13.858025217447853
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.514648938179016,
      "eval_nll_ablated": 2.523477633794149,
      "label_use_gap": 0.008828695615132887,
      "perplexity_per_byte_labeled": 12.362268096883284,
      "perplexity_per_byte_ablated": 12.47189401387667
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3897199551264445,
      "eval_nll_ablated": 2.4119556029637654,
      "label_use_gap": 0.022235647837320904,
      "perplexity_per_byte_labeled": 10.910438102916798,
      "perplexity_per_byte_ablated": 11.155756056629862
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7288443565368654,
      "eval_nll_ablated": 2.735476922988892,
      "label_use_gap": 0.006632566452026367,
      "perplexity_per_byte_labeled": 15.31517790393461,
      "perplexity_per_byte_ablated": 15.417094449621214
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6217588742574054,
      "eval_nll_ablated": 2.6281398216883343,
      "label_use_gap": 0.006380947430928874,
      "perplexity_per_byte_labeled": 13.75990425484349,
      "perplexity_per_byte_ablated": 13.847986204829045
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.511163258552551,
      "eval_nll_ablated": 2.5250561555226643,
      "label_use_gap": 0.013892896970113089,
      "perplexity_per_byte_labeled": 12.319252204204892,
      "perplexity_per_byte_ablated": 12.49159671605391
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.376479482650757,
      "eval_nll_ablated": 2.4112589518229166,
      "label_use_gap": 0.03477946917215968,
      "perplexity_per_byte_labeled": 10.766930895647308,
      "perplexity_per_byte_ablated": 11.147987092888911
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7280009428660077,
      "eval_nll_ablated": 2.737621712684631,
      "label_use_gap": 0.009620769818623476,
      "perplexity_per_byte_labeled": 15.30226631918774,
      "perplexity_per_byte_ablated": 15.450196360564348
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.623139476776123,
      "eval_nll_ablated": 2.630717174212138,
      "label_use_gap": 0.007577697436014663,
      "perplexity_per_byte_labeled": 13.778914332976383,
      "perplexity_per_byte_ablated": 13.88372338089137
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5148298343022666,
      "eval_nll_ablated": 2.526538054148356,
      "label_use_gap": 0.011708219846089385,
      "perplexity_per_byte_labeled": 12.364504585536558,
      "perplexity_per_byte_ablated": 12.510121718758038
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3825168291727703,
      "eval_nll_ablated": 2.4211869796117145,
      "label_use_gap": 0.03867015043894417,
      "perplexity_per_byte_labeled": 10.832131208841549,
      "perplexity_per_byte_ablated": 11.259215846001691
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.728957978884379,
      "eval_nll_ablated": 2.737689177195231,
      "label_use_gap": 0.008731198310851784,
      "perplexity_per_byte_labeled": 15.316918149264152,
      "perplexity_per_byte_ablated": 15.451238735661754
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6223525365193683,
      "eval_nll_ablated": 2.629420256614685,
      "label_use_gap": 0.007067720095316865,
      "perplexity_per_byte_labeled": 13.76807541594282,
      "perplexity_per_byte_ablated": 13.865729006852904
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5185343424479165,
      "eval_nll_ablated": 2.529818805058797,
      "label_use_gap": 0.011284462610880563,
      "perplexity_per_byte_labeled": 12.41039393975411,
      "perplexity_per_byte_ablated": 12.55123171092535
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.383985177675883,
      "eval_nll_ablated": 2.4141174952189126,
      "label_use_gap": 0.030132317543029696,
      "perplexity_per_byte_labeled": 10.848048235498881,
      "perplexity_per_byte_ablated": 11.179899687810133
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7295082728068034,
      "eval_nll_ablated": 2.737262002627055,
      "label_use_gap": 0.007753729820251465,
      "perplexity_per_byte_labeled": 15.325349275818052,
      "perplexity_per_byte_ablated": 15.444639768982778
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.622507619857788,
      "eval_nll_ablated": 2.6281593958536784,
      "label_use_gap": 0.005651775995890507,
      "perplexity_per_byte_labeled": 13.770210780617399,
      "perplexity_per_byte_ablated": 13.848257270253633
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.512163345019023,
      "eval_nll_ablated": 2.5203959385554,
      "label_use_gap": 0.008232593536376953,
      "perplexity_per_byte_labeled": 12.331578684356973,
      "perplexity_per_byte_ablated": 12.433518598513771
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3909834384918214,
      "eval_nll_ablated": 2.414607350031535,
      "label_use_gap": 0.02362391153971366,
      "perplexity_per_byte_labeled": 10.924231972296026,
      "perplexity_per_byte_ablated": 11.18537755704761
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.728364594777425,
      "eval_nll_ablated": 2.736958400408427,
      "label_use_gap": 0.00859380563100176,
      "perplexity_per_byte_labeled": 15.307832029512996,
      "perplexity_per_byte_ablated": 15.439951453810462
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6216348091761272,
      "eval_nll_ablated": 2.627242596944173,
      "label_use_gap": 0.005607787768045824,
      "perplexity_per_byte_labeled": 13.758197237096569,
      "perplexity_per_byte_ablated": 13.835567021181838
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.514535125096639,
      "eval_nll_ablated": 2.522599935531616,
      "label_use_gap": 0.00806481043497742,
      "perplexity_per_byte_labeled": 12.360861189109778,
      "perplexity_per_byte_ablated": 12.46095225665327
    },
    {
      "train_per_category": 100,
      "seed": 22189,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3847110668818154,
      "eval_nll_ablated": 2.4127330859502156,
      "label_use_gap": 0.028022019068400184,
      "perplexity_per_byte_labeled": 10.855925575310545,
      "perplexity_per_byte_ablated": 11.16443283975343
    }
  ]
}