{
  "experiment": "H1 real-text conditional byte-model pilot v13 (v11 data and structure, larger model)",
  "completed_utc": "2026-11-19T02:32:38.393106+00:00",
  "smoke_test_not_protocol": false,
  "protocol": {
    "dataset": "SetFit/20_newsgroups",
    "revision": "f1b91292074e7cfb69be58b642d583ec262f30ed",
    "dataset_split": "train only; official test unused",
    "selected_data_sha256": "72325b82d18ed20a73b97086b6447e027c820a42c3ce4440cf75e384875095c1",
    "categories": [
      "comp.graphics",
      "comp.os.ms-windows.misc",
      "comp.sys.ibm.pc.hardware",
      "comp.sys.mac.hardware",
      "rec.autos",
      "rec.motorcycles",
      "rec.sport.baseball",
      "rec.sport.hockey"
    ],
    "sizes_train_per_category": [
      100
    ],
    "subsets_nested": true,
    "validation_per_category": 80,
    "validation_identical_across_sizes": true,
    "validation_messages": 640,
    "bytes_per_example": 96,
    "seed_values": [
      13121,
      14129,
      15137,
      16141,
      17149
    ],
    "steps": [
      100,
      200,
      400,
      800
    ],
    "total_updates": 800,
    "batch_size": 24,
    "conditions": [
      "none",
      "hierarchy",
      "scrambled_1",
      "scrambled_2",
      "scrambled_3",
      "scrambled_4",
      "scrambled_5"
    ],
    "permutation_seeds_selected": [
      11011,
      22022,
      33033,
      44044,
      55055
    ],
    "permutation_seeds_rejected": [],
    "permutation_seeds_primary": [
      11011,
      22022,
      33033,
      44044,
      55055
    ],
    "permutation_seeds_fallback": [
      66066,
      77077,
      88088,
      99099
    ],
    "scrambled_permutations": {
      "scrambled_1": [
        7,
        2,
        3,
        6,
        0,
        1,
        5,
        4
      ],
      "scrambled_2": [
        4,
        7,
        5,
        3,
        1,
        2,
        6,
        0
      ],
      "scrambled_3": [
        0,
        5,
        2,
        7,
        6,
        1,
        4,
        3
      ],
      "scrambled_4": [
        6,
        7,
        3,
        0,
        2,
        5,
        4,
        1
      ],
      "scrambled_5": [
        2,
        6,
        5,
        1,
        3,
        4,
        7,
        0
      ]
    },
    "permutation_rejection_rule": "rejected if categories 0-3 all fall in the same code//4 group (comp/rec split preserved), same as v11",
    "same_batch_sequence_per_size_and_seed_across_conditions": true,
    "batch_generator_seed": "seed + 30000, per (size, seed)",
    "evaluations": "each checkpoint: (a) eval_nll_labeled with the condition label prefix; (b) eval_nll_ablated with dummy prefix [256, 256] for all conditions; label_use_gap = ablated - labeled",
    "metric": "validation NLL in nats per byte, lower is better",
    "model": "randomly initialized 4-layer causal transformer width 128, 4 heads, FF 256, dropout 0.2 in training only (eval mode at every evaluation)",
    "dropout": 0.2,
    "weight_decay": 0.1,
    "learning_rate": 0.001,
    "regularisation_chosen_before_run": true,
    "v9_reference": "v9 used dropout 0 and weight decay 0.01",
    "replication_of": "v11 (data, training, rule, permutation rejection) with a larger model",
    "model_size_differs_from_v10_v12": true,
    "n_layers": 4,
    "d_model": 128,
    "n_heads": 4,
    "ff": 256,
    "v11_model": "2 layers, width 64, 4 heads, FF 128",
    "reference_v10_v11_n100_paired_diff": {
      "200": -0.0006,
      "400": -0.0006
    },
    "reference_v10_v11_n100_provenance": "pooled v10+v11, 10 seeds, exploratory; FACTS.md; unpaired with v13 (other model and seeds), indicative only",
    "preregistered_cells": [
      {
        "train_per_category": 100,
        "step": 200
      },
      {
        "train_per_category": 100,
        "step": 400
      }
    ],
    "new_seeds_and_permutations_vs_v10_v11_v12": true,
    "v10_seeds_not_reused": [
      2027,
      3037,
      4049,
      5051,
      6067
    ],
    "v11_seeds_not_reused": [
      7079,
      8081,
      9091,
      10093,
      11099
    ],
    "v12_seeds_not_reused": [
      12101,
      13103,
      14107,
      15109,
      16111
    ],
    "primary_checkpoint": 800,
    "checkpoint_selection": "none; all checkpoints are reported and the primary criterion is fixed in advance at 800 updates",
    "limitations": "one model larger than v11 only (4 layers, width 128), not a scaling law; a single regularisation setting (dropout 0.2, weight decay 0.1) not tuned, which may under- or over-regularise the 4-layer model; five seeds, five permutations, two cells that share seeds across checkpoints and are not independent; comparison with the v10+v11 reference is unpaired (other model, other seeds) and purely indicative; no significance test is run or claimed; no data-savings percentage is estimated; labels present at labeled evaluation; cells chosen AFTER seeing v10; one historical text dataset (20 Newsgroups); not a pretrained-LLM or general data-savings result"
  },
  "environment": {
    "device": "cuda",
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition MIG 2g.48gb",
    "torch": "2.13.0+cu130",
    "cuda": "13.0",
    "python": "3.12.12",
    "code_sha256": "7ee35f7dfb89a35a055862afce15119859c97486783ad6a3461461bc7b20f22c"
  },
  "summary": {
    "100": {
      "100": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.7247722641626995,
          "hierarchy": 2.7248336442311603,
          "scrambled_1": 2.7254915046691894,
          "scrambled_2": 2.7259195629755655,
          "scrambled_3": 2.7258565839131674,
          "scrambled_4": 2.7263783661524457,
          "scrambled_5": 2.7257852602005004,
          "scrambled_mean": 2.7258862555821737
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0010526113510133327,
          "min": -0.002326663335164625,
          "max": -0.00021600564320900517,
          "per_seed": {
            "13121": -0.002326663335164625,
            "14129": -0.00021979331970234384,
            "15137": -0.0019469547271731003,
            "16141": -0.0005536397298175899,
            "17149": -0.00021600564320900517
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.4,
          "per_seed": {
            "13121": 0,
            "14129": 3,
            "15137": 0,
            "16141": 2,
            "17149": 2
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.4,
          "per_seed": {
            "13121": 1,
            "14129": 4,
            "15137": 1,
            "16141": 3,
            "17149": 3
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0020415306091307707,
          "per_seed": {
            "13121": 0.002884968121846576,
            "14129": 0.000996390978495132,
            "15137": 0.0020882447560626183,
            "16141": 0.002307327588399133,
            "17149": 0.0019307216008503936
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 5
      },
      "200": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.6213463624318445,
          "hierarchy": 2.6207231108347573,
          "scrambled_1": 2.620677002271017,
          "scrambled_2": 2.6213617595036824,
          "scrambled_3": 2.6225534423192345,
          "scrambled_4": 2.621213763554891,
          "scrambled_5": 2.6206019846598307,
          "scrambled_mean": 2.621281590461731
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0005584796269733517,
          "min": -0.001609964370727468,
          "max": 0.0010308567682906222,
          "per_seed": {
            "13121": -0.001220272382100429,
            "14129": 0.0010308567682906222,
            "15137": -0.0016054407755534683,
            "16141": 0.000612422625223985,
            "17149": -0.001609964370727468
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.8,
          "per_seed": {
            "13121": 0,
            "14129": 4,
            "15137": 0,
            "16141": 5,
            "17149": 0
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.8,
          "per_seed": {
            "13121": 1,
            "14129": 5,
            "15137": 1,
            "16141": 6,
            "17149": 1
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.002425923347473091,
          "per_seed": {
            "13121": 0.004793922106424819,
            "14129": 0.003842385609944543,
            "15137": 0.0008001645406086055,
            "16141": 0.0014824549357097538,
            "17149": 0.0012106895446777344
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 3
      },
      "400": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.520520060857137,
          "hierarchy": 2.521273760795593,
          "scrambled_1": 2.523128995895386,
          "scrambled_2": 2.5206036376953125,
          "scrambled_3": 2.5214713271458944,
          "scrambled_4": 2.521636422475179,
          "scrambled_5": 2.521179943084717,
          "scrambled_mean": 2.5216040652592975
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0003303044637044117,
          "min": -0.005470970471700198,
          "max": 0.003641635576884461,
          "per_seed": {
            "13121": -0.001222434043884224,
            "14129": 0.003498953183491693,
            "15137": -0.005470970471700198,
            "16141": -0.0020987065633137902,
            "17149": 0.003641635576884461
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 2.2,
          "per_seed": {
            "13121": 2,
            "14129": 5,
            "15137": 0,
            "16141": 0,
            "17149": 4
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 3.2,
          "per_seed": {
            "13121": 3,
            "14129": 6,
            "15137": 1,
            "16141": 1,
            "17149": 5
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.006232612927754566,
          "per_seed": {
            "13121": 0.009360051155089977,
            "14129": 0.004660026232401471,
            "15137": 0.0032359997431434806,
            "16141": 0.004076512654622277,
            "17149": 0.009830474853515625
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 3
      },
      "800": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.392498024304708,
          "hierarchy": 2.3980363321304323,
          "scrambled_1": 2.397642774581909,
          "scrambled_2": 2.3984916671117147,
          "scrambled_3": 2.39637176990509,
          "scrambled_4": 2.3965268818537395,
          "scrambled_5": 2.3984972381591794,
          "scrambled_mean": 2.3975060663223267
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": 0.0005302658081055789,
          "min": -0.007348182996114172,
          "max": 0.004295624097188533,
          "per_seed": {
            "13121": -0.007348182996114172,
            "14129": 0.0021806017557781665,
            "15137": 0.0016228564580278437,
            "16141": 0.0019004297256475233,
            "17149": 0.004295624097188533
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 3.6,
          "per_seed": {
            "13121": 0,
            "14129": 4,
            "15137": 5,
            "16141": 4,
            "17149": 5
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 4.6,
          "per_seed": {
            "13121": 1,
            "14129": 5,
            "15137": 6,
            "16141": 5,
            "17149": 6
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.008632890383402448,
          "per_seed": {
            "13121": 0.012221368153889856,
            "14129": 0.012662267684936612,
            "15137": 0.003993280728658011,
            "16141": 0.008161958058674923,
            "17149": 0.006125577290852835
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 1
      }
    }
  },
  "replication_cells": {
    "cells": {
      "100_200": {
        "train_per_category": 100,
        "step": 200,
        "mean_paired_diff": -0.0005584796269733517,
        "seeds_hierarchy_better": 3,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.002425923347473091,
        "sign_negative": true,
        "wins_at_least_4_of_5": false,
        "material": false
      },
      "100_400": {
        "train_per_category": 100,
        "step": 400,
        "mean_paired_diff": -0.0003303044637044117,
        "seeds_hierarchy_better": 3,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.006232612927754566,
        "sign_negative": true,
        "wins_at_least_4_of_5": false,
        "material": false
      }
    },
    "missing_cells": [],
    "n_cells_sign_negative": 2,
    "n_cells_wins_at_least_4_of_5": 0,
    "n_cells_material": 0
  },
  "reference_v10_v11_n100": {
    "200": -0.0006,
    "400": -0.0006
  },
  "summary_label_use": {
    "100": {
      "100": {
        "none": 0.0,
        "hierarchy": 0.00454258124033613,
        "scrambled_1": 0.0037308502197264557,
        "scrambled_2": 0.0035426759719848987,
        "scrambled_3": 0.003373047510782978,
        "scrambled_4": 0.003748318354288749,
        "scrambled_5": 0.0032646115620931226,
        "scrambled_mean": 0.003531900723775241
      },
      "200": {
        "none": 0.0,
        "hierarchy": 0.005813953081766865,
        "scrambled_1": 0.005650900204976317,
        "scrambled_2": 0.005605988502502513,
        "scrambled_3": 0.004254771868387852,
        "scrambled_4": 0.0053953043619791256,
        "scrambled_5": 0.005287292798360266,
        "scrambled_mean": 0.005238851547241214
      },
      "400": {
        "none": 0.0,
        "hierarchy": 0.010699227650960364,
        "scrambled_1": 0.009049100875854598,
        "scrambled_2": 0.007443974812825438,
        "scrambled_3": 0.006863166491190675,
        "scrambled_4": 0.008758562405904158,
        "scrambled_5": 0.009269231160481794,
        "scrambled_mean": 0.008276807149251332
      },
      "800": {
        "none": 0.0,
        "hierarchy": 0.0190960566202798,
        "scrambled_1": 0.016343166033427003,
        "scrambled_2": 0.01560540199279794,
        "scrambled_3": 0.011368015607197978,
        "scrambled_4": 0.016297510464986155,
        "scrambled_5": 0.017005154291788747,
        "scrambled_mean": 0.015323849678039564
      }
    }
  },
  "elapsed_seconds": 193.8605860369862,
  "runs": [
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7298994859059653,
      "eval_nll_ablated": 2.7298994859059653,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.331345926111574,
      "perplexity_per_byte_ablated": 15.331345926111574
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.617868932088216,
      "eval_nll_ablated": 2.617868932088216,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.70648299321038,
      "perplexity_per_byte_ablated": 13.70648299321038
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5117752075195314,
      "eval_nll_ablated": 2.5117752075195314,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.326793265002074,
      "perplexity_per_byte_ablated": 12.326793265002074
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3811978022257487,
      "eval_nll_ablated": 2.3811978022257487,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.817852754786419,
      "perplexity_per_byte_ablated": 10.817852754786419
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7276830514272055,
      "eval_nll_ablated": 2.731044554710388,
      "label_use_gap": 0.003361503283182632,
      "perplexity_per_byte_labeled": 15.297402632833684,
      "perplexity_per_byte_ablated": 15.34891142699563
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6167829116185506,
      "eval_nll_ablated": 2.6207911650339764,
      "label_use_gap": 0.0040082534154257665,
      "perplexity_per_byte_labeled": 13.691605552177636,
      "perplexity_per_byte_ablated": 13.746595109312644
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.516134524345398,
      "eval_nll_ablated": 2.5307974100112913,
      "label_use_gap": 0.014662885665893466,
      "perplexity_per_byte_labeled": 12.38064695966514,
      "perplexity_per_byte_ablated": 12.563520420352809
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3892205794652304,
      "eval_nll_ablated": 2.4148401021957397,
      "label_use_gap": 0.02561952273050938,
      "perplexity_per_byte_labeled": 10.90499105584958,
      "perplexity_per_byte_ablated": 11.187981280880884
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7287935972213746,
      "eval_nll_ablated": 2.7321377913157145,
      "label_use_gap": 0.0033441940943399473,
      "perplexity_per_byte_labeled": 15.314400535717091,
      "perplexity_per_byte_ablated": 15.365700594413578
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.616865118344625,
      "eval_nll_ablated": 2.62228410243988,
      "label_use_gap": 0.005418984095255208,
      "perplexity_per_byte_labeled": 13.692731140509615,
      "perplexity_per_byte_ablated": 13.767133242614156
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5211190462112425,
      "eval_nll_ablated": 2.5312881390253703,
      "label_use_gap": 0.010169092814127811,
      "perplexity_per_byte_labeled": 12.442512622433293,
      "perplexity_per_byte_ablated": 12.569687217331396
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.403304696083069,
      "eval_nll_ablated": 2.420928684870402,
      "label_use_gap": 0.01762398878733329,
      "perplexity_per_byte_labeled": 11.059664886954003,
      "perplexity_per_byte_ablated": 11.25630802531093
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7308213313420615,
      "eval_nll_ablated": 2.733521644274394,
      "label_use_gap": 0.002700312932332416,
      "perplexity_per_byte_labeled": 15.345485573666235,
      "perplexity_per_byte_ablated": 15.386979184467444
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6167903741200766,
      "eval_nll_ablated": 2.622122597694397,
      "label_use_gap": 0.005332223574320505,
      "perplexity_per_byte_labeled": 13.6917077261862,
      "perplexity_per_byte_ablated": 13.764909964803568
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5117589950561525,
      "eval_nll_ablated": 2.5220570961634317,
      "label_use_gap": 0.010298101107279134,
      "perplexity_per_byte_labeled": 12.326593418937689,
      "perplexity_per_byte_ablated": 12.454189796834058
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.391083327929179,
      "eval_nll_ablated": 2.413326382637024,
      "label_use_gap": 0.022243054707844934,
      "perplexity_per_byte_labeled": 10.925323242183563,
      "perplexity_per_byte_ablated": 11.171058626101537
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.728572384516398,
      "eval_nll_ablated": 2.7305737257003786,
      "label_use_gap": 0.002001341183980543,
      "perplexity_per_byte_labeled": 15.311013170427424,
      "perplexity_per_byte_ablated": 15.341686415229772
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6215842962265015,
      "eval_nll_ablated": 2.6242599487304688,
      "label_use_gap": 0.002675652503967285,
      "perplexity_per_byte_labeled": 13.757502287524714,
      "perplexity_per_byte_ablated": 13.79436187269918
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.520490352312724,
      "eval_nll_ablated": 2.526737395922343,
      "label_use_gap": 0.006247043609619141,
      "perplexity_per_byte_labeled": 12.43469254913909,
      "perplexity_per_byte_ablated": 12.512615757189016
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.397669816017151,
      "eval_nll_ablated": 2.4128344217936197,
      "label_use_gap": 0.015164605776468942,
      "perplexity_per_byte_labeled": 10.997520254954754,
      "perplexity_per_byte_ablated": 11.16556425429683
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7314573526382446,
      "eval_nll_ablated": 2.733789825439453,
      "label_use_gap": 0.0023324728012084073,
      "perplexity_per_byte_labeled": 15.355248733751104,
      "perplexity_per_byte_ablated": 15.391106235845863
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6178643226623537,
      "eval_nll_ablated": 2.6213257471720377,
      "label_use_gap": 0.003461424509684008,
      "perplexity_per_byte_labeled": 13.7064198143388,
      "perplexity_per_byte_ablated": 13.753945758104964
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5180007298787435,
      "eval_nll_ablated": 2.526750016212463,
      "label_use_gap": 0.008749286333719741,
      "perplexity_per_byte_labeled": 12.40377336412775,
      "perplexity_per_byte_ablated": 12.512773671026489
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.395941456158956,
      "eval_nll_ablated": 2.4144229094187417,
      "label_use_gap": 0.018481453259785674,
      "perplexity_per_byte_labeled": 10.978528998996236,
      "perplexity_per_byte_ablated": 11.183314709398953
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7304039080937703,
      "eval_nll_ablated": 2.732840418815613,
      "label_use_gap": 0.0024365107218424775,
      "perplexity_per_byte_labeled": 15.339081347960807,
      "perplexity_per_byte_ablated": 15.376500752005956
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6169118086496987,
      "eval_nll_ablated": 2.620873467127482,
      "label_use_gap": 0.003961658477783292,
      "perplexity_per_byte_labeled": 13.693370473229061,
      "perplexity_per_byte_ablated": 13.747726529427199
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.515415668487549,
      "eval_nll_ablated": 2.5236672004063925,
      "label_use_gap": 0.008251531918843646,
      "perplexity_per_byte_labeled": 12.37175025718069,
      "perplexity_per_byte_ablated": 12.474258492679178
    },
    {
      "train_per_category": 100,
      "seed": 13121,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3948445161183676,
      "eval_nll_ablated": 2.4169186274210612,
      "label_use_gap": 0.022074111302693655,
      "perplexity_per_byte_labeled": 10.966492813644205,
      "perplexity_per_byte_ablated": 11.211259966480391
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7173297723134358,
      "eval_nll_ablated": 2.7173297723134358,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.139841398795305,
      "perplexity_per_byte_ablated": 15.139841398795305
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.622290253639221,
      "eval_nll_ablated": 2.622290253639221,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.767217927255542,
      "perplexity_per_byte_ablated": 13.767217927255542
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.537023162841797,
      "eval_nll_ablated": 2.537023162841797,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.641981780257675,
      "perplexity_per_byte_ablated": 12.641981780257675
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4104756832122805,
      "eval_nll_ablated": 2.4104756832122805,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 11.1392586433338,
      "perplexity_per_byte_ablated": 11.1392586433338
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7167911847432453,
      "eval_nll_ablated": 2.7220231453577677,
      "label_use_gap": 0.005231960614522357,
      "perplexity_per_byte_labeled": 15.13168946386573,
      "perplexity_per_byte_ablated": 15.21106533181367
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.622184236844381,
      "eval_nll_ablated": 2.6243733882904055,
      "label_use_gap": 0.0021891514460246064,
      "perplexity_per_byte_labeled": 13.765758448303032,
      "perplexity_per_byte_ablated": 13.795926787799589
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.536315600077311,
      "eval_nll_ablated": 2.544229261080424,
      "label_use_gap": 0.00791366100311297,
      "perplexity_per_byte_labeled": 12.633039948507308,
      "perplexity_per_byte_ablated": 12.73341016823027
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.408042144775391,
      "eval_nll_ablated": 2.420428196589152,
      "label_use_gap": 0.012386051813761068,
      "perplexity_per_byte_labeled": 11.112183786480893,
      "perplexity_per_byte_ablated": 11.250675784606921
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.71672728061676,
      "eval_nll_ablated": 2.7207201957702636,
      "label_use_gap": 0.003992915153503418,
      "perplexity_per_byte_labeled": 15.13072251736456,
      "perplexity_per_byte_ablated": 15.191258986650782
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.620211927096049,
      "eval_nll_ablated": 2.622534688313802,
      "label_use_gap": 0.0023227612177527668,
      "perplexity_per_byte_labeled": 13.738634865568157,
      "perplexity_per_byte_ablated": 13.770583524006987
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.532663877805074,
      "eval_nll_ablated": 2.5389161984125774,
      "label_use_gap": 0.006252320607503581,
      "perplexity_per_byte_labeled": 12.58699172399617,
      "perplexity_per_byte_ablated": 12.665936167552928
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3998989423116046,
      "eval_nll_ablated": 2.4099998076756797,
      "label_use_gap": 0.010100865364075062,
      "perplexity_per_byte_labeled": 11.022062460203859,
      "perplexity_per_byte_ablated": 11.133959003734
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.717212184270223,
      "eval_nll_ablated": 2.7212674617767334,
      "label_use_gap": 0.0040552775065103575,
      "perplexity_per_byte_labeled": 15.138061239135466,
      "perplexity_per_byte_ablated": 15.199574921596357
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.621687873204549,
      "eval_nll_ablated": 2.6226982911427816,
      "label_use_gap": 0.0010104179382324219,
      "perplexity_per_byte_labeled": 13.758927321836241,
      "perplexity_per_byte_ablated": 13.772836614729199
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5305398384730022,
      "eval_nll_ablated": 2.5358531951904295,
      "label_use_gap": 0.0053133567174272756,
      "perplexity_per_byte_labeled": 12.560284831787461,
      "perplexity_per_byte_ablated": 12.627199719478233
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4125612099965412,
      "eval_nll_ablated": 2.420027796427409,
      "label_use_gap": 0.007466586430867839,
      "perplexity_per_byte_labeled": 11.162514107108588,
      "perplexity_per_byte_ablated": 11.246171913938493
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7176728089650473,
      "eval_nll_ablated": 2.7218714475631716,
      "label_use_gap": 0.004198638598124305,
      "perplexity_per_byte_labeled": 15.145035810180485,
      "perplexity_per_byte_ablated": 15.208758021760724
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.623581560452779,
      "eval_nll_ablated": 2.623506943384806,
      "label_use_gap": -7.461706797284862e-05,
      "perplexity_per_byte_labeled": 13.785007112745042,
      "perplexity_per_byte_ablated": 13.783978554306788
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5318493366241457,
      "eval_nll_ablated": 2.535505175590515,
      "label_use_gap": 0.0036558389663694513,
      "perplexity_per_byte_labeled": 12.57674327535134,
      "perplexity_per_byte_ablated": 12.622805971083446
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4054172039031982,
      "eval_nll_ablated": 2.4137881278991697,
      "label_use_gap": 0.008370923995971502,
      "perplexity_per_byte_labeled": 11.083053210826344,
      "perplexity_per_byte_ablated": 11.17621800056198
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7167661984761557,
      "eval_nll_ablated": 2.72206867535909,
      "label_use_gap": 0.005302476882934393,
      "perplexity_per_byte_labeled": 15.131311384154689,
      "perplexity_per_byte_ablated": 15.211757907404708
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6205463647842406,
      "eval_nll_ablated": 2.6215699513753257,
      "label_use_gap": 0.001023586591085035,
      "perplexity_per_byte_labeled": 13.743230351260477,
      "perplexity_per_byte_ablated": 13.757304939617317
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5351998647054037,
      "eval_nll_ablated": 2.5410693486531577,
      "label_use_gap": 0.005869483947753995,
      "perplexity_per_byte_labeled": 12.618952679275724,
      "perplexity_per_byte_ablated": 12.693237212137197
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4060792128245034,
      "eval_nll_ablated": 2.4161305745442707,
      "label_use_gap": 0.010051361719767282,
      "perplexity_per_byte_labeled": 11.090392720069461,
      "perplexity_per_byte_ablated": 11.202428381146529
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.716676417986552,
      "eval_nll_ablated": 2.7202572504679363,
      "label_use_gap": 0.0035808324813841885,
      "perplexity_per_byte_labeled": 15.129952948591692,
      "perplexity_per_byte_ablated": 15.184227892298098
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6197391748428345,
      "eval_nll_ablated": 2.6210745096206667,
      "label_use_gap": 0.0013353347778322089,
      "perplexity_per_byte_labeled": 13.732141429993455,
      "perplexity_per_byte_ablated": 13.750490684491298
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5338303168614704,
      "eval_nll_ablated": 2.543177302678426,
      "label_use_gap": 0.009346985816955478,
      "perplexity_per_byte_labeled": 12.601682248880937,
      "perplexity_per_byte_ablated": 12.720022193448603
    },
    {
      "train_per_category": 100,
      "seed": 14129,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4053511460622152,
      "eval_nll_ablated": 2.418188460667928,
      "label_use_gap": 0.012837314605712802,
      "perplexity_per_byte_labeled": 11.082321112440422,
      "perplexity_per_byte_ablated": 11.225505439898768
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.744893193244934,
      "eval_nll_ablated": 2.744893193244934,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.562951615051551,
      "perplexity_per_byte_ablated": 15.562951615051551
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.624481272697449,
      "eval_nll_ablated": 2.624481272697449,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.797415233469302,
      "perplexity_per_byte_ablated": 13.797415233469302
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.519789473215739,
      "eval_nll_ablated": 2.519789473215739,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.425980386496946,
      "perplexity_per_byte_ablated": 12.425980386496946
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.39290558497111,
      "eval_nll_ablated": 2.39290558497111,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.945250139847662,
      "perplexity_per_byte_ablated": 10.945250139847662
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7417819420496623,
      "eval_nll_ablated": 2.748065455754598,
      "label_use_gap": 0.006283513704935739,
      "perplexity_per_byte_labeled": 15.514606608964083,
      "perplexity_per_byte_ablated": 15.61239977280077
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.620181608200073,
      "eval_nll_ablated": 2.6285506169001263,
      "label_use_gap": 0.008369008700053193,
      "perplexity_per_byte_labeled": 13.738218331641272,
      "perplexity_per_byte_ablated": 13.853676059857536
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.511488127708435,
      "eval_nll_ablated": 2.5263834238052367,
      "label_use_gap": 0.014895296096801847,
      "perplexity_per_byte_labeled": 12.323254999426737,
      "perplexity_per_byte_ablated": 12.508187423898397
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.389537477493286,
      "eval_nll_ablated": 2.413029925028483,
      "label_use_gap": 0.02349244753519697,
      "perplexity_per_byte_labeled": 10.908447373632363,
      "perplexity_per_byte_ablated": 11.167747371623916
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7433226982752483,
      "eval_nll_ablated": 2.7480445941289267,
      "label_use_gap": 0.004721895853678326,
      "perplexity_per_byte_labeled": 15.538529260438903,
      "perplexity_per_byte_ablated": 15.612074076158171
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.621406984329224,
      "eval_nll_ablated": 2.627902603149414,
      "label_use_gap": 0.006495618820190341,
      "perplexity_per_byte_labeled": 13.755063134944088,
      "perplexity_per_byte_ablated": 13.84470159537508
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.516337823867798,
      "eval_nll_ablated": 2.529276450475057,
      "label_use_gap": 0.012938626607259085,
      "perplexity_per_byte_labeled": 12.383164195146762,
      "perplexity_per_byte_ablated": 12.544426338504625
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.389496080080668,
      "eval_nll_ablated": 2.410900608698527,
      "label_use_gap": 0.021404528617858798,
      "perplexity_per_byte_labeled": 10.907995801482441,
      "perplexity_per_byte_ablated": 11.14399300403327
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.743859116236369,
      "eval_nll_ablated": 2.7476693471272786,
      "label_use_gap": 0.003810230890909594,
      "perplexity_per_byte_labeled": 15.546866642584448,
      "perplexity_per_byte_ablated": 15.606216791204625
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.622189919153849,
      "eval_nll_ablated": 2.6290491422017417,
      "label_use_gap": 0.006859223047892549,
      "perplexity_per_byte_labeled": 13.765836669824841,
      "perplexity_per_byte_ablated": 13.86058418968944
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5175367752710978,
      "eval_nll_ablated": 2.527561910947164,
      "label_use_gap": 0.01002513567606611,
      "perplexity_per_byte_labeled": 12.398019911096995,
      "perplexity_per_byte_ablated": 12.522936851244475
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.388521162668864,
      "eval_nll_ablated": 2.408760174115499,
      "label_use_gap": 0.02023901144663487,
      "perplexity_per_byte_labeled": 10.89736658859194,
      "perplexity_per_byte_ablated": 11.120165525690803
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7436484177907308,
      "eval_nll_ablated": 2.7484819253285724,
      "label_use_gap": 0.004833507537841619,
      "perplexity_per_byte_labeled": 15.543591287016591,
      "perplexity_per_byte_ablated": 15.618903216432066
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6220902681350706,
      "eval_nll_ablated": 2.6272361199061076,
      "label_use_gap": 0.005145851771036991,
      "perplexity_per_byte_labeled": 13.764464958523703,
      "perplexity_per_byte_ablated": 13.835477407977798
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.518190550804138,
      "eval_nll_ablated": 2.531510345141093,
      "label_use_gap": 0.01331979433695496,
      "perplexity_per_byte_labeled": 12.406128083346532,
      "perplexity_per_byte_ablated": 12.572480589044453
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3873496055603027,
      "eval_nll_ablated": 2.4012568314870197,
      "label_use_gap": 0.013907225926717004,
      "perplexity_per_byte_labeled": 10.884607176949588,
      "perplexity_per_byte_ablated": 11.037039365696964
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7428630034128827,
      "eval_nll_ablated": 2.7478304386138914,
      "label_use_gap": 0.00496743520100873,
      "perplexity_per_byte_labeled": 15.531387919913694,
      "perplexity_per_byte_ablated": 15.608731022373103
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.621858318646749,
      "eval_nll_ablated": 2.6292906284332274,
      "label_use_gap": 0.007432309786478353,
      "perplexity_per_byte_labeled": 13.761272668158924,
      "perplexity_per_byte_ablated": 13.863931734108274
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.517775789896647,
      "eval_nll_ablated": 2.5323018074035644,
      "label_use_gap": 0.014526017506917288,
      "perplexity_per_byte_labeled": 12.400983573348801,
      "perplexity_per_byte_ablated": 12.58243517179499
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.38550279935201,
      "eval_nll_ablated": 2.409222785631816,
      "label_use_gap": 0.02371998627980565,
      "perplexity_per_byte_labeled": 10.864523967446507,
      "perplexity_per_byte_ablated": 11.125311032419907
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7449512481689453,
      "eval_nll_ablated": 2.7480207761128743,
      "label_use_gap": 0.003069527943929007,
      "perplexity_per_byte_labeled": 15.563855147251946,
      "perplexity_per_byte_ablated": 15.61170223195553
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6213897546132405,
      "eval_nll_ablated": 2.62805548508962,
      "label_use_gap": 0.006665730476379306,
      "perplexity_per_byte_labeled": 13.754826141154615,
      "perplexity_per_byte_ablated": 13.846818362020121
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5149545510609945,
      "eval_nll_ablated": 2.5268221616744997,
      "label_use_gap": 0.011867610613505164,
      "perplexity_per_byte_labeled": 12.366046742636163,
      "perplexity_per_byte_ablated": 12.513676443429418
    },
    {
      "train_per_category": 100,
      "seed": 15137,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.388703457514445,
      "eval_nll_ablated": 2.411281172434489,
      "label_use_gap": 0.022577714920044123,
      "perplexity_per_byte_labeled": 10.899353303429882,
      "perplexity_per_byte_ablated": 11.148234810732129
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7109619140625,
      "eval_nll_ablated": 2.7109619140625,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.043739341706306,
      "perplexity_per_byte_ablated": 15.043739341706306
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.624336814880371,
      "eval_nll_ablated": 2.624336814880371,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.795422232939078,
      "perplexity_per_byte_ablated": 13.795422232939078
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5163605848948163,
      "eval_nll_ablated": 2.5163605848948163,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.38344605188924,
      "perplexity_per_byte_ablated": 12.38344605188924
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.388168986638387,
      "eval_nll_ablated": 2.388168986638387,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.893529472993844,
      "perplexity_per_byte_ablated": 10.893529472993844
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7157696962356566,
      "eval_nll_ablated": 2.7192208051681517,
      "label_use_gap": 0.003451108932495117,
      "perplexity_per_byte_labeled": 15.116240508786358,
      "perplexity_per_byte_ablated": 15.168498423443374
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.628415807088216,
      "eval_nll_ablated": 2.634587597846985,
      "label_use_gap": 0.006171790758768836,
      "perplexity_per_byte_labeled": 13.851808574274163,
      "perplexity_per_byte_ablated": 13.937563396628084
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.526391537984212,
      "eval_nll_ablated": 2.5343388398488362,
      "label_use_gap": 0.007947301864624112,
      "perplexity_per_byte_labeled": 12.508288917981583,
      "perplexity_per_byte_ablated": 12.60809212363017
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4154060522715253,
      "eval_nll_ablated": 2.4326420625050864,
      "label_use_gap": 0.01723601023356114,
      "perplexity_per_byte_labeled": 11.194314911825087,
      "perplexity_per_byte_ablated": 11.388932636688624
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7165287176767987,
      "eval_nll_ablated": 2.71977485815684,
      "label_use_gap": 0.0032461404800412375,
      "perplexity_per_byte_labeled": 15.127718414879354,
      "perplexity_per_byte_ablated": 15.176904903931566
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6267873366673786,
      "eval_nll_ablated": 2.6322043816248577,
      "label_use_gap": 0.005417044957479078,
      "perplexity_per_byte_labeled": 13.829269670685697,
      "perplexity_per_byte_ablated": 13.904386718745625
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.530106616020203,
      "eval_nll_ablated": 2.536337129275004,
      "label_use_gap": 0.006230513254801284,
      "perplexity_per_byte_labeled": 12.55484461288235,
      "perplexity_per_byte_ablated": 12.633311930649592
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.411092662811279,
      "eval_nll_ablated": 2.4239410638809202,
      "label_use_gap": 0.012848401069641024,
      "perplexity_per_byte_labeled": 11.14613345925721,
      "perplexity_per_byte_ablated": 11.290267414938402
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7153741041819255,
      "eval_nll_ablated": 2.7190956592559816,
      "label_use_gap": 0.0037215550740561376,
      "perplexity_per_byte_labeled": 15.110261826796307,
      "perplexity_per_byte_ablated": 15.16660026664768
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6280815839767455,
      "eval_nll_ablated": 2.6329497973124187,
      "label_use_gap": 0.0048682133356732216,
      "perplexity_per_byte_labeled": 13.847179753285536,
      "perplexity_per_byte_ablated": 13.914755130640037
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5271175384521483,
      "eval_nll_ablated": 2.531447656949361,
      "label_use_gap": 0.004330118497212876,
      "perplexity_per_byte_labeled": 12.517373238800689,
      "perplexity_per_byte_ablated": 12.571692467673952
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4153565406799316,
      "eval_nll_ablated": 2.428697450955709,
      "label_use_gap": 0.013340910275777329,
      "perplexity_per_byte_labeled": 11.193760677197634,
      "perplexity_per_byte_ablated": 11.344096210754381
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.716330822308858,
      "eval_nll_ablated": 2.719644379615784,
      "label_use_gap": 0.0033135573069258406,
      "perplexity_per_byte_labeled": 15.124725005678215,
      "perplexity_per_byte_ablated": 15.174924772707081
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6278119802474977,
      "eval_nll_ablated": 2.6334973494211833,
      "label_use_gap": 0.005685369173685562,
      "perplexity_per_byte_labeled": 13.8434470051885,
      "perplexity_per_byte_ablated": 13.92237627044994
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.527361257870992,
      "eval_nll_ablated": 2.5319305260976157,
      "label_use_gap": 0.004569268226623624,
      "perplexity_per_byte_labeled": 12.520424337522801,
      "perplexity_per_byte_ablated": 12.577764415968208
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4090351819992066,
      "eval_nll_ablated": 2.417758345603943,
      "label_use_gap": 0.00872316360473624,
      "perplexity_per_byte_labeled": 11.12322407942242,
      "perplexity_per_byte_ablated": 11.220678219112829
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7176814317703246,
      "eval_nll_ablated": 2.7212093512217206,
      "label_use_gap": 0.0035279194513959666,
      "perplexity_per_byte_labeled": 15.145166403438234,
      "perplexity_per_byte_ablated": 15.198691691524447
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6282697916030884,
      "eval_nll_ablated": 2.6336864868799847,
      "label_use_gap": 0.005416695276896366,
      "perplexity_per_byte_labeled": 13.849786143381998,
      "perplexity_per_byte_ablated": 13.925009762356343
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.530971161524455,
      "eval_nll_ablated": 2.53586589495341,
      "label_use_gap": 0.004894733428955078,
      "perplexity_per_byte_labeled": 12.565703540691244,
      "perplexity_per_byte_ablated": 12.627360082940061
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.414846587181091,
      "eval_nll_ablated": 2.4277098576227822,
      "label_use_gap": 0.01286327044169111,
      "perplexity_per_byte_labeled": 11.188053835010859,
      "perplexity_per_byte_ablated": 11.332898387326583
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7157016038894652,
      "eval_nll_ablated": 2.719725171724955,
      "label_use_gap": 0.00402356783548985,
      "perplexity_per_byte_labeled": 15.115211243547463,
      "perplexity_per_byte_ablated": 15.176150836413452
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6280662298202513,
      "eval_nll_ablated": 2.632715113957723,
      "label_use_gap": 0.004648884137471665,
      "perplexity_per_byte_labeled": 13.84696714315283,
      "perplexity_per_byte_ablated": 13.911489952382688
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5268946488698325,
      "eval_nll_ablated": 2.5346799453099567,
      "label_use_gap": 0.0077852964401241564,
      "perplexity_per_byte_labeled": 12.514583557614788,
      "perplexity_per_byte_ablated": 12.612393546285523
    },
    {
      "train_per_category": 100,
      "seed": 16141,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4171971400578816,
      "eval_nll_ablated": 2.4305779854456584,
      "label_use_gap": 0.01338084538777684,
      "perplexity_per_byte_labeled": 11.214382878920835,
      "perplexity_per_byte_ablated": 11.365449246203067
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7207769552866616,
      "eval_nll_ablated": 2.7207769552866616,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.192121259635208,
      "perplexity_per_byte_ablated": 15.192121259635208
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6177545388539634,
      "eval_nll_ablated": 2.6177545388539634,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.704915153967368,
      "perplexity_per_byte_ablated": 13.704915153967368
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5176518758138022,
      "eval_nll_ablated": 2.5176518758138022,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.399447012045695,
      "perplexity_per_byte_ablated": 12.399447012045695
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.389742064476013,
      "eval_nll_ablated": 2.389742064476013,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.91067932827342,
      "perplexity_per_byte_ablated": 10.91067932827342
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7221423467000325,
      "eval_nll_ablated": 2.7265271663665773,
      "label_use_gap": 0.004384819666544804,
      "perplexity_per_byte_labeled": 15.212878619289503,
      "perplexity_per_byte_ablated": 15.279730808930166
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6160509904225666,
      "eval_nll_ablated": 2.6243825515111285,
      "label_use_gap": 0.008331561088561923,
      "perplexity_per_byte_labeled": 13.681588042328046,
      "perplexity_per_byte_ablated": 13.796053203501012
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.51603901386261,
      "eval_nll_ablated": 2.5241160074869793,
      "label_use_gap": 0.008076993624369422,
      "perplexity_per_byte_labeled": 12.37946453456469,
      "perplexity_per_byte_ablated": 12.479858284734929
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3879754066467287,
      "eval_nll_ablated": 2.404721657435099,
      "label_use_gap": 0.016746250788370443,
      "perplexity_per_byte_labeled": 10.89142090774394,
      "perplexity_per_byte_ablated": 11.075347112595177
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.722085229555766,
      "eval_nll_ablated": 2.725434335072835,
      "label_use_gap": 0.0033491055170693507,
      "perplexity_per_byte_labeled": 15.212009727921227,
      "perplexity_per_byte_ablated": 15.263041761759666
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.618113644917806,
      "eval_nll_ablated": 2.62671373685201,
      "label_use_gap": 0.00860009193420419,
      "perplexity_per_byte_labeled": 13.709837555882896,
      "perplexity_per_byte_ablated": 13.828251876446448
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5154176155726113,
      "eval_nll_ablated": 2.5250725666681926,
      "label_use_gap": 0.009654951095581232,
      "perplexity_per_byte_labeled": 12.371774346054265,
      "perplexity_per_byte_ablated": 12.491801719147663
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3844214916229247,
      "eval_nll_ablated": 2.4041595379511516,
      "label_use_gap": 0.019738046328226844,
      "perplexity_per_byte_labeled": 10.852782422963116,
      "perplexity_per_byte_ablated": 11.069123193648643
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7223310788472492,
      "eval_nll_ablated": 2.725757082303365,
      "label_use_gap": 0.003426003456115989,
      "perplexity_per_byte_labeled": 15.21575004949374,
      "perplexity_per_byte_ablated": 15.267968661246554
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6180590470631917,
      "eval_nll_ablated": 2.6280189116795856,
      "label_use_gap": 0.009959864616393865,
      "perplexity_per_byte_labeled": 13.709089048598868,
      "perplexity_per_byte_ablated": 13.846311945915236
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5160650412241616,
      "eval_nll_ablated": 2.5233182032903034,
      "label_use_gap": 0.007253162066141794,
      "perplexity_per_byte_labeled": 12.379786743557055,
      "perplexity_per_byte_ablated": 12.469905772027541
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3849360942840576,
      "eval_nll_ablated": 2.3996735413869223,
      "label_use_gap": 0.014737447102864731,
      "perplexity_per_byte_labeled": 10.858368730919862,
      "perplexity_per_byte_ablated": 11.019578357103509
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.723058485984802,
      "eval_nll_ablated": 2.7255766789118447,
      "label_use_gap": 0.002518192927042584,
      "perplexity_per_byte_labeled": 15.226822121146732,
      "perplexity_per_byte_ablated": 15.265214516354446
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6176991065343223,
      "eval_nll_ablated": 2.6255407094955445,
      "label_use_gap": 0.007841602961222272,
      "perplexity_per_byte_labeled": 13.704155479785344,
      "perplexity_per_byte_ablated": 13.812040468774521
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.509465138117472,
      "eval_nll_ablated": 2.5159890254338584,
      "label_use_gap": 0.006523887316386201,
      "perplexity_per_byte_labeled": 12.29835038222029,
      "perplexity_per_byte_ablated": 12.378845720050734
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3823870420455933,
      "eval_nll_ablated": 2.3930612007776895,
      "label_use_gap": 0.010674158732096206,
      "perplexity_per_byte_labeled": 10.830725428878793,
      "perplexity_per_byte_ablated": 10.94695352630988
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7231238444646197,
      "eval_nll_ablated": 2.725735131899516,
      "label_use_gap": 0.002611287434896248,
      "perplexity_per_byte_labeled": 15.227817355616175,
      "perplexity_per_byte_ablated": 15.267633526846664
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.617530020078023,
      "eval_nll_ablated": 2.627172525723775,
      "label_use_gap": 0.009642505645751864,
      "perplexity_per_byte_labeled": 13.701838488590132,
      "perplexity_per_byte_ablated": 13.834597580081121
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.506234566370646,
      "eval_nll_ablated": 2.5159878571828207,
      "label_use_gap": 0.009753290812174686,
      "perplexity_per_byte_labeled": 12.258683786333576,
      "perplexity_per_byte_ablated": 12.378831258459824
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3802643537521364,
      "eval_nll_ablated": 2.3966358343760175,
      "label_use_gap": 0.016371480623881052,
      "perplexity_per_byte_labeled": 10.807759558117086,
      "perplexity_per_byte_ablated": 10.986154897711083
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7211931228637694,
      "eval_nll_ablated": 2.7244057416915894,
      "label_use_gap": 0.0032126188278200907,
      "perplexity_per_byte_labeled": 15.198445043716635,
      "perplexity_per_byte_ablated": 15.247350369440449
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6169029553731282,
      "eval_nll_ablated": 2.626727811495463,
      "label_use_gap": 0.009824856122334857,
      "perplexity_per_byte_labeled": 13.693249242569724,
      "perplexity_per_byte_ablated": 13.82844650553085
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5148045301437376,
      "eval_nll_ablated": 2.523899261156718,
      "label_use_gap": 0.009094731012980528,
      "perplexity_per_byte_labeled": 12.36419171611086,
      "perplexity_per_byte_ablated": 12.47715361437381
    },
    {
      "train_per_category": 100,
      "seed": 17149,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3863899310429892,
      "eval_nll_ablated": 2.4005457162857056,
      "label_use_gap": 0.014155785242716323,
      "perplexity_per_byte_labeled": 10.874166507434442,
      "perplexity_per_byte_ablated": 11.029193549196831
    }
  ]
}