{
  "experiment": "H1 real-text conditional byte-model pilot v15 (v13 larger model, stronger regularisation)",
  "completed_utc": "2026-11-19T02:49:24.973974+00:00",
  "smoke_test_not_protocol": false,
  "protocol": {
    "dataset": "SetFit/20_newsgroups",
    "revision": "f1b91292074e7cfb69be58b642d583ec262f30ed",
    "dataset_split": "train only; official test unused",
    "selected_data_sha256": "72325b82d18ed20a73b97086b6447e027c820a42c3ce4440cf75e384875095c1",
    "categories": [
      "comp.graphics",
      "comp.os.ms-windows.misc",
      "comp.sys.ibm.pc.hardware",
      "comp.sys.mac.hardware",
      "rec.autos",
      "rec.motorcycles",
      "rec.sport.baseball",
      "rec.sport.hockey"
    ],
    "sizes_train_per_category": [
      100
    ],
    "subsets_nested": true,
    "validation_per_category": 80,
    "validation_identical_across_sizes": true,
    "validation_messages": 640,
    "bytes_per_example": 96,
    "seed_values": [
      23197,
      24203,
      25219,
      26227,
      27239
    ],
    "steps": [
      100,
      200,
      400,
      800
    ],
    "total_updates": 800,
    "batch_size": 24,
    "conditions": [
      "none",
      "hierarchy",
      "scrambled_1",
      "scrambled_2",
      "scrambled_3",
      "scrambled_4",
      "scrambled_5"
    ],
    "permutation_seeds_selected": [
      13013,
      24024,
      35035,
      46046,
      57057
    ],
    "permutation_seeds_rejected": [],
    "permutation_seeds_primary": [
      13013,
      24024,
      35035,
      46046,
      57057
    ],
    "permutation_seeds_fallback": [
      68068,
      79079,
      81081,
      92929
    ],
    "scrambled_permutations": {
      "scrambled_1": [
        3,
        0,
        6,
        4,
        7,
        2,
        5,
        1
      ],
      "scrambled_2": [
        5,
        4,
        6,
        3,
        7,
        0,
        2,
        1
      ],
      "scrambled_3": [
        5,
        2,
        4,
        6,
        0,
        1,
        3,
        7
      ],
      "scrambled_4": [
        7,
        5,
        4,
        3,
        2,
        0,
        6,
        1
      ],
      "scrambled_5": [
        1,
        5,
        6,
        0,
        2,
        4,
        7,
        3
      ]
    },
    "permutation_rejection_rule": "rejected if categories 0-3 all fall in the same code//4 group (comp/rec split preserved), same as v11",
    "same_batch_sequence_per_size_and_seed_across_conditions": true,
    "batch_generator_seed": "seed + 30000, per (size, seed)",
    "evaluations": "each checkpoint: (a) eval_nll_labeled with the condition label prefix; (b) eval_nll_ablated with dummy prefix [256, 256] for all conditions; label_use_gap = ablated - labeled",
    "metric": "validation NLL in nats per byte, lower is better",
    "model": "randomly initialized 4-layer causal transformer width 128, 4 heads, FF 256, dropout 0.3 in training only (eval mode at every evaluation)",
    "dropout": 0.3,
    "weight_decay": 0.3,
    "learning_rate": 0.001,
    "regularisation_chosen_before_run": true,
    "regularisation_changed_vs_v13": "dropout 0.2 -> 0.3, weight decay 0.1 -> 0.3 (the only change)",
    "v9_reference": "v9 used dropout 0 and weight decay 0.01",
    "v13_regularisation": "dropout 0.2, weight decay 0.1",
    "replication_of": "v13 (data, model, training, rule, permutation rejection) with stronger regularisation",
    "model_size_differs_from_v10_v12": true,
    "n_layers": 4,
    "d_model": 128,
    "n_heads": 4,
    "ff": 256,
    "v11_model": "2 layers, width 64, 4 heads, FF 128",
    "reference_v13_none_nll_100_800": 2.3925,
    "reference_v13_none_nll_100_800_provenance": "v13.json summary[100][800] mean_eval_nll_labeled_over_seeds[none]; v13 seeds 13121..17149; unpaired with v15 (other seeds), indicative only",
    "preregistered_cells": [
      {
        "train_per_category": 100,
        "step": 200
      },
      {
        "train_per_category": 100,
        "step": 400
      }
    ],
    "new_seeds_and_permutations_vs_v10_v11_v12_v13_v14": true,
    "v10_seeds_not_reused": [
      2027,
      3037,
      4049,
      5051,
      6067
    ],
    "v11_seeds_not_reused": [
      7079,
      8081,
      9091,
      10093,
      11099
    ],
    "v12_seeds_not_reused": [
      12101,
      13103,
      14107,
      15109,
      16111
    ],
    "v13_seeds_not_reused": [
      13121,
      14129,
      15137,
      16141,
      17149
    ],
    "v14_seeds_not_reused": [
      18163,
      19169,
      20173,
      21179,
      22189
    ],
    "primary_checkpoint": 800,
    "checkpoint_selection": "none; all checkpoints are reported and the primary criterion is fixed in advance at 800 updates",
    "limitations": "one larger model only (4 layers, width 128), not a scaling law; one stronger regularisation setting (dropout 0.3, weight decay 0.3) not tuned, which may under- or over-regularise the 4-layer model, and dropout and weight decay change together so their effects cannot be separated; five seeds, five permutations, two cells that share seeds across checkpoints and are not independent; comparison with v13 is unpaired (other seeds) and purely indicative; no significance test is run or claimed; no data-savings percentage is estimated; labels present at labeled evaluation; cells chosen AFTER seeing v10; one historical text dataset (20 Newsgroups); not a pretrained-LLM or general data-savings result"
  },
  "environment": {
    "device": "cuda",
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition MIG 2g.48gb",
    "torch": "2.13.0+cu130",
    "cuda": "13.0",
    "python": "3.12.12",
    "code_sha256": "847995254eae8bd30be28c6fea9af9f4770b090667ce51244c9e256549616da2"
  },
  "summary": {
    "100": {
      "100": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.736931120554606,
          "hierarchy": 2.737987585067749,
          "scrambled_1": 2.7381410026550292,
          "scrambled_2": 2.736790897051493,
          "scrambled_3": 2.7369245306650796,
          "scrambled_4": 2.7372852325439454,
          "scrambled_5": 2.738304006258647,
          "scrambled_mean": 2.7374891338348393
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": 0.0004984512329101243,
          "min": -0.0013148864110310576,
          "max": 0.00249821503957115,
          "per_seed": {
            "23197": 8.63615671793383e-05,
            "24203": 0.0007806857426961855,
            "25219": 0.00249821503957115,
            "26227": -0.0013148864110310576,
            "27239": 0.0004418802261350052
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 3.0,
          "per_seed": {
            "23197": 2,
            "24203": 5,
            "25219": 4,
            "26227": 0,
            "27239": 4
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 4.0,
          "per_seed": {
            "23197": 3,
            "24203": 6,
            "25219": 5,
            "26227": 1,
            "27239": 5
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0020134909947712386,
          "per_seed": {
            "23197": 0.0014459212621051876,
            "24203": 0.0012770175933840555,
            "25219": 0.005309502283731948,
            "26227": 0.0008558750152585226,
            "27239": 0.0011791388193764796
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 1
      },
      "200": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.63094170888265,
          "hierarchy": 2.6313582468032837,
          "scrambled_1": 2.632531391779582,
          "scrambled_2": 2.6320469999313354,
          "scrambled_3": 2.631868235270182,
          "scrambled_4": 2.6318970902760825,
          "scrambled_5": 2.6323194122314453,
          "scrambled_mean": 2.632132625897725
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.000774379094441624,
          "min": -0.0019428269068400006,
          "max": -3.4605662027509254e-05,
          "per_seed": {
            "23197": -6.59894943235706e-05,
            "24203": -0.0019428269068400006,
            "25219": -0.0007116031646732246,
            "26227": -3.4605662027509254e-05,
            "27239": -0.001116870244343815
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.4,
          "per_seed": {
            "23197": 2,
            "24203": 0,
            "25219": 1,
            "26227": 3,
            "27239": 1
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.4,
          "per_seed": {
            "23197": 3,
            "24203": 1,
            "25219": 2,
            "26227": 4,
            "27239": 2
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.0016790819168090998,
          "per_seed": {
            "23197": 0.0010854800542197118,
            "24203": 0.001081053415934008,
            "25219": 0.0022640784581504647,
            "26227": 0.0013814449310305399,
            "27239": 0.0025833527247107746
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 5
      },
      "400": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.5415885400772096,
          "hierarchy": 2.5418070809046425,
          "scrambled_1": 2.5427054341634117,
          "scrambled_2": 2.542712666193644,
          "scrambled_3": 2.54264919757843,
          "scrambled_4": 2.5435246912638343,
          "scrambled_5": 2.542392071088155,
          "scrambled_mean": 2.542796812057495
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": -0.0009897311528523467,
          "min": -0.0020158799489338186,
          "max": 4.478295644094743e-05,
          "per_seed": {
            "23197": -0.0009714921315508995,
            "24203": -0.0005565770467121922,
            "25219": -0.0020158799489338186,
            "26227": 4.478295644094743e-05,
            "27239": -0.0014494895935057706
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 1.2,
          "per_seed": {
            "23197": 1,
            "24203": 2,
            "25219": 0,
            "26227": 2,
            "27239": 1
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 2.2,
          "per_seed": {
            "23197": 2,
            "24203": 3,
            "25219": 1,
            "26227": 3,
            "27239": 2
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.003190253575642998,
          "per_seed": {
            "23197": 0.0019508997599282374,
            "24203": 0.0020646174748741863,
            "25219": 0.0033664544423421816,
            "26227": 0.004907528559366714,
            "27239": 0.0036617676417036726
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 4
      },
      "800": {
        "mean_eval_nll_labeled_over_seeds": {
          "none": 2.3717096583048503,
          "hierarchy": 2.3852739493052164,
          "scrambled_1": 2.3844287618001303,
          "scrambled_2": 2.384412693977356,
          "scrambled_3": 2.384105725288391,
          "scrambled_4": 2.384821783701579,
          "scrambled_5": 2.3830504210789996,
          "scrambled_mean": 2.384163877169291
        },
        "paired_diff_hierarchy_minus_scrambled_mean": {
          "mean": 0.0011100721359253996,
          "min": -0.0023784796396890684,
          "max": 0.0040800809860233045,
          "per_seed": {
            "23197": 0.0009771251678465376,
            "24203": 0.0006116024653115559,
            "25219": 0.002260031700134668,
            "26227": 0.0040800809860233045,
            "27239": -0.0023784796396890684
          }
        },
        "n_scrambled_better_than_hierarchy": {
          "mean_over_seeds": 3.0,
          "per_seed": {
            "23197": 3,
            "24203": 3,
            "25219": 4,
            "26227": 5,
            "27239": 0
          }
        },
        "hierarchy_rank_among_6_codes": {
          "mean_over_seeds": 4.0,
          "per_seed": {
            "23197": 4,
            "24203": 4,
            "25219": 5,
            "26227": 6,
            "27239": 1
          }
        },
        "between_permutation_range_of_scrambled": {
          "mean_over_seeds": 0.00421679337819425,
          "per_seed": {
            "23197": 0.004582548141479403,
            "24203": 0.004527600606282878,
            "25219": 0.005847152074178297,
            "26227": 0.002222196261087994,
            "27239": 0.0039044698079426787
          }
        },
        "seeds_where_hierarchy_beats_scrambled_mean": 1
      }
    }
  },
  "replication_cells": {
    "cells": {
      "100_200": {
        "train_per_category": 100,
        "step": 200,
        "mean_paired_diff": -0.000774379094441624,
        "seeds_hierarchy_better": 5,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.0016790819168090998,
        "sign_negative": true,
        "wins_at_least_4_of_5": true,
        "material": false
      },
      "100_400": {
        "train_per_category": 100,
        "step": 400,
        "mean_paired_diff": -0.0009897311528523467,
        "seeds_hierarchy_better": 4,
        "n_seeds": 5,
        "mean_between_permutation_range": 0.003190253575642998,
        "sign_negative": true,
        "wins_at_least_4_of_5": true,
        "material": false
      }
    },
    "missing_cells": [],
    "n_cells_sign_negative": 2,
    "n_cells_wins_at_least_4_of_5": 2,
    "n_cells_material": 0
  },
  "reference_v13_none_nll_100_800": 2.3925,
  "p5_vs_v13_indicative": {
    "v15_none_nll_100_800": 2.3717096583048503,
    "v13_reference_none_nll_100_800": 2.3925,
    "v15_lower_than_v13": true,
    "unpaired_indicative_only": true
  },
  "summary_label_use": {
    "100": {
      "100": {
        "none": 0.0,
        "hierarchy": 0.0029738426208495206,
        "scrambled_1": 0.0022954893112182794,
        "scrambled_2": 0.002505569458007795,
        "scrambled_3": 0.002779362996419188,
        "scrambled_4": 0.0027068551381429274,
        "scrambled_5": 0.0028300364812215762,
        "scrambled_mean": 0.002623462677001953
      },
      "200": {
        "none": 0.0,
        "hierarchy": 0.0033956925074259737,
        "scrambled_1": 0.0034508800506590554,
        "scrambled_2": 0.003475708961486923,
        "scrambled_3": 0.003265368143717495,
        "scrambled_4": 0.0035482470194499348,
        "scrambled_5": 0.003490123748779261,
        "scrambled_mean": 0.0034460655848185338
      },
      "400": {
        "none": 0.0,
        "hierarchy": 0.0068941068649290395,
        "scrambled_1": 0.00657632986704515,
        "scrambled_2": 0.005666650136311802,
        "scrambled_3": 0.006276639302571585,
        "scrambled_4": 0.006259188652038538,
        "scrambled_5": 0.007015980084737183,
        "scrambled_mean": 0.0063589576085408515
      },
      "800": {
        "none": 0.0,
        "hierarchy": 0.011648750305175603,
        "scrambled_1": 0.010355386734008843,
        "scrambled_2": 0.009981385866800974,
        "scrambled_3": 0.011410191853841133,
        "scrambled_4": 0.011741213798522932,
        "scrambled_5": 0.011728151639302542,
        "scrambled_mean": 0.011043265978495285
      }
    }
  },
  "elapsed_seconds": 195.60177642200142,
  "runs": [
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7207547187805177,
      "eval_nll_ablated": 2.7207547187805177,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.191783443693415,
      "perplexity_per_byte_ablated": 15.191783443693415
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6262373765309652,
      "eval_nll_ablated": 2.6262373765309652,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.821666214641613,
      "perplexity_per_byte_ablated": 13.821666214641613
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5316051721572874,
      "eval_nll_ablated": 2.5316051721572874,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.573672856393559,
      "perplexity_per_byte_ablated": 12.573672856393559
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.359808087348938,
      "eval_nll_ablated": 2.359808087348938,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.588919109885635,
      "perplexity_per_byte_ablated": 10.588919109885635
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.724966319402059,
      "eval_nll_ablated": 2.7279647827148437,
      "label_use_gap": 0.0029984633127848603,
      "perplexity_per_byte_labeled": 15.25590009036887,
      "perplexity_per_byte_ablated": 15.301712996928655
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.628299840291341,
      "eval_nll_ablated": 2.6345754067103067,
      "label_use_gap": 0.006275566418965539,
      "perplexity_per_byte_labeled": 13.850202317540901,
      "perplexity_per_byte_ablated": 13.937393482923474
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5400487820307416,
      "eval_nll_ablated": 2.5540995836257934,
      "label_use_gap": 0.014050801595051787,
      "perplexity_per_byte_labeled": 12.680289526020035,
      "perplexity_per_byte_ablated": 12.859715344688336
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3813910484313965,
      "eval_nll_ablated": 2.3984416643778483,
      "label_use_gap": 0.017050615946451853,
      "perplexity_per_byte_labeled": 10.819943465789015,
      "perplexity_per_byte_ablated": 11.006011949664023
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7244757334391276,
      "eval_nll_ablated": 2.726323628425598,
      "label_use_gap": 0.001847894986470422,
      "perplexity_per_byte_labeled": 15.248417595486204,
      "perplexity_per_byte_ablated": 15.27662112046313
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6284398714701336,
      "eval_nll_ablated": 2.6346861124038696,
      "label_use_gap": 0.006246240933736047,
      "perplexity_per_byte_labeled": 13.852141913496725,
      "perplexity_per_byte_ablated": 13.938936517145223
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.541173505783081,
      "eval_nll_ablated": 2.5572325865427654,
      "label_use_gap": 0.01605908075968454,
      "perplexity_per_byte_labeled": 12.694559372149648,
      "perplexity_per_byte_ablated": 12.900068049938781
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3803823073705037,
      "eval_nll_ablated": 2.399704845746358,
      "label_use_gap": 0.019322538375854403,
      "perplexity_per_byte_labeled": 10.809034447650848,
      "perplexity_per_byte_ablated": 11.019923323344676
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7241514921188354,
      "eval_nll_ablated": 2.7262733300526936,
      "label_use_gap": 0.0021218379338581173,
      "perplexity_per_byte_labeled": 15.243474229897679,
      "perplexity_per_byte_ablated": 15.27585275060133
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6278153896331786,
      "eval_nll_ablated": 2.63251158396403,
      "label_use_gap": 0.004696194330851444,
      "perplexity_per_byte_labeled": 13.84349420291895,
      "perplexity_per_byte_ablated": 13.908658835038842
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5399985392888387,
      "eval_nll_ablated": 2.5530343294143676,
      "label_use_gap": 0.013035790125528912,
      "perplexity_per_byte_labeled": 12.679652449510494,
      "perplexity_per_byte_ablated": 12.846023772556835
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3825729290644326,
      "eval_nll_ablated": 2.3986140727996825,
      "label_use_gap": 0.016041143735249896,
      "perplexity_per_byte_labeled": 10.832738907274585,
      "perplexity_per_byte_ablated": 11.007909642399357
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7251747687657675,
      "eval_nll_ablated": 2.726905926068624,
      "label_use_gap": 0.0017311573028564453,
      "perplexity_per_byte_labeled": 15.259080504501647,
      "perplexity_per_byte_ablated": 15.285519251363874
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6289008696873983,
      "eval_nll_ablated": 2.6336573521296183,
      "label_use_gap": 0.004756482442219934,
      "perplexity_per_byte_labeled": 13.858529198374498,
      "perplexity_per_byte_ablated": 13.924604066583017
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5406453609466553,
      "eval_nll_ablated": 2.5558317422866823,
      "label_use_gap": 0.015186381340027033,
      "perplexity_per_byte_labeled": 12.68785657634585,
      "perplexity_per_byte_ablated": 12.882009715118333
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.379228973388672,
      "eval_nll_ablated": 2.3967989524205526,
      "label_use_gap": 0.017569979031880845,
      "perplexity_per_byte_labeled": 10.796575207125409,
      "perplexity_per_byte_ablated": 10.987947083979943
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7250003814697266,
      "eval_nll_ablated": 2.7280617634455364,
      "label_use_gap": 0.0030613819758098515,
      "perplexity_per_byte_labeled": 15.256419746720317,
      "perplexity_per_byte_ablated": 15.303197040196585
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6280608415603637,
      "eval_nll_ablated": 2.6341045538584393,
      "label_use_gap": 0.006043712298075654,
      "perplexity_per_byte_labeled": 13.846892532296218,
      "perplexity_per_byte_ablated": 13.930832566188835
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.541949439048767,
      "eval_nll_ablated": 2.5563480218251544,
      "label_use_gap": 0.014398582776387503,
      "perplexity_per_byte_labeled": 12.704413325570455,
      "perplexity_per_byte_ablated": 12.888662150258956
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3818950255711875,
      "eval_nll_ablated": 2.401245625813802,
      "label_use_gap": 0.01935060024261448,
      "perplexity_per_byte_labeled": 10.82539784427519,
      "perplexity_per_byte_ablated": 11.036915688933483
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7255974133809406,
      "eval_nll_ablated": 2.728191916147868,
      "label_use_gap": 0.002594502766927409,
      "perplexity_per_byte_labeled": 15.265531035754497,
      "perplexity_per_byte_ablated": 15.305188922267286
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.62861217657725,
      "eval_nll_ablated": 2.635845653216044,
      "label_use_gap": 0.007233476638794034,
      "perplexity_per_byte_labeled": 13.8545289139332,
      "perplexity_per_byte_ablated": 13.955108657231387
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.54133452574412,
      "eval_nll_ablated": 2.55849449634552,
      "label_use_gap": 0.01715997060139962,
      "perplexity_per_byte_labeled": 12.696603614182632,
      "perplexity_per_byte_ablated": 12.916357047728619
    },
    {
      "train_per_category": 100,
      "seed": 23197,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3779903809229532,
      "eval_nll_ablated": 2.3979426542917888,
      "label_use_gap": 0.019952273368835538,
      "perplexity_per_byte_labeled": 10.783210928574185,
      "perplexity_per_byte_ablated": 11.000521208775329
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.753946590423584,
      "eval_nll_ablated": 2.753946590423584,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.70448892730071,
      "perplexity_per_byte_ablated": 15.70448892730071
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.628938659032186,
      "eval_nll_ablated": 2.628938659032186,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.859052913007979,
      "perplexity_per_byte_ablated": 13.859052913007979
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.551418654123942,
      "eval_nll_ablated": 2.551418654123942,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.825285527014012,
      "perplexity_per_byte_ablated": 12.825285527014012
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.388635492324829,
      "eval_nll_ablated": 2.388635492324829,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.898612551988865,
      "perplexity_per_byte_ablated": 10.898612551988865
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7550405899683637,
      "eval_nll_ablated": 2.755904698371887,
      "label_use_gap": 0.000864108403523467,
      "perplexity_per_byte_labeled": 15.721679032307225,
      "perplexity_per_byte_ablated": 15.735270138525403
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.629080073038737,
      "eval_nll_ablated": 2.6321668068567914,
      "label_use_gap": 0.0030867338180544657,
      "perplexity_per_byte_labeled": 13.861012915790065,
      "perplexity_per_byte_ablated": 13.903864274454987
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5574913024902344,
      "eval_nll_ablated": 2.562377349535624,
      "label_use_gap": 0.004886047045389752,
      "perplexity_per_byte_labeled": 12.903405935030168,
      "perplexity_per_byte_ablated": 12.966606859083283
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4193360805511475,
      "eval_nll_ablated": 2.427051242192586,
      "label_use_gap": 0.007715161641438595,
      "perplexity_per_byte_labeled": 11.238395448140949,
      "perplexity_per_byte_ablated": 11.325436823000263
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.754943672815959,
      "eval_nll_ablated": 2.7556395212809246,
      "label_use_gap": 0.0006958484649657315,
      "perplexity_per_byte_labeled": 15.720155405778357,
      "perplexity_per_byte_ablated": 15.73109805855902
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.630816348393758,
      "eval_nll_ablated": 2.63431183497111,
      "label_use_gap": 0.0034954865773517696,
      "perplexity_per_byte_labeled": 13.885100356074577,
      "perplexity_per_byte_ablated": 13.933720463956572
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.557001519203186,
      "eval_nll_ablated": 2.5635721683502197,
      "label_use_gap": 0.006570649147033869,
      "perplexity_per_byte_labeled": 12.897087609888523,
      "perplexity_per_byte_ablated": 12.98210886411941
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4204351902008057,
      "eval_nll_ablated": 2.4268529812494912,
      "label_use_gap": 0.006417791048685562,
      "perplexity_per_byte_labeled": 11.250754467739341,
      "perplexity_per_byte_ablated": 11.323191653786811
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.753994663556417,
      "eval_nll_ablated": 2.755024798711141,
      "label_use_gap": 0.0010301351547239435,
      "perplexity_per_byte_labeled": 15.705243909430022,
      "perplexity_per_byte_ablated": 15.721430769189853
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.631423282623291,
      "eval_nll_ablated": 2.63535483678182,
      "label_use_gap": 0.0039315541585289715,
      "perplexity_per_byte_labeled": 13.89353025670002,
      "perplexity_per_byte_ablated": 13.948260941184285
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.55906613667806,
      "eval_nll_ablated": 2.5632097562154135,
      "label_use_gap": 0.004143619537353427,
      "perplexity_per_byte_labeled": 12.923742669125009,
      "perplexity_per_byte_ablated": 12.97740484278037
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4173429091771443,
      "eval_nll_ablated": 2.4224605798721313,
      "label_use_gap": 0.00511767069498692,
      "perplexity_per_byte_labeled": 11.216017708787131,
      "perplexity_per_byte_ablated": 11.273564721661577
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.753666655222575,
      "eval_nll_ablated": 2.7547937393188477,
      "label_use_gap": 0.0011270840962729345,
      "perplexity_per_byte_labeled": 15.70009330330975,
      "perplexity_per_byte_ablated": 15.717798604589376
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.630342229207357,
      "eval_nll_ablated": 2.6333423852920532,
      "label_use_gap": 0.0030001560846963926,
      "perplexity_per_byte_labeled": 13.878518723952977,
      "perplexity_per_byte_ablated": 13.920218968692279
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.557927306493123,
      "eval_nll_ablated": 2.5617132584253945,
      "label_use_gap": 0.003785951932271381,
      "perplexity_per_byte_labeled": 12.909033098312454,
      "perplexity_per_byte_ablated": 12.957998709352541
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4162174781163532,
      "eval_nll_ablated": 2.422907527287801,
      "label_use_gap": 0.006690049171447665,
      "perplexity_per_byte_labeled": 11.203401954491737,
      "perplexity_per_byte_ablated": 11.278604538461959
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.754671239852905,
      "eval_nll_ablated": 2.7550941705703735,
      "label_use_gap": 0.00042293071746835054,
      "perplexity_per_byte_labeled": 15.715873300581285,
      "perplexity_per_byte_ablated": 15.72252143190232
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.63137211004893,
      "eval_nll_ablated": 2.6350375493367513,
      "label_use_gap": 0.0036654392878214814,
      "perplexity_per_byte_labeled": 13.892819307180535,
      "perplexity_per_byte_ablated": 13.943836035127799
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5588436841964723,
      "eval_nll_ablated": 2.5628110567728677,
      "label_use_gap": 0.003967372576395345,
      "perplexity_per_byte_labeled": 12.920868070239543,
      "perplexity_per_byte_ablated": 12.972231790019016
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.420745078722636,
      "eval_nll_ablated": 2.4269009828567505,
      "label_use_gap": 0.006155904134114376,
      "perplexity_per_byte_labeled": 11.254241487676648,
      "perplexity_per_byte_ablated": 11.323735198230896
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.754023289680481,
      "eval_nll_ablated": 2.754395270347595,
      "label_use_gap": 0.0003719806671140802,
      "perplexity_per_byte_labeled": 15.70569349612556,
      "perplexity_per_byte_ablated": 15.711536797199877
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.631160529454549,
      "eval_nll_ablated": 2.6347258647282916,
      "label_use_gap": 0.003565335273742587,
      "perplexity_per_byte_labeled": 13.889880167157358,
      "perplexity_per_byte_ablated": 13.939490633285379
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5574007511138914,
      "eval_nll_ablated": 2.5620102802912395,
      "label_use_gap": 0.004609529177348026,
      "perplexity_per_byte_labeled": 12.902237566762714,
      "perplexity_per_byte_ablated": 12.961848089953614
    },
    {
      "train_per_category": 100,
      "seed": 24203,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.4188817342122397,
      "eval_nll_ablated": 2.426276715596517,
      "label_use_gap": 0.007394981384277077,
      "perplexity_per_byte_labeled": 11.233290484112565,
      "perplexity_per_byte_ablated": 11.316668367108237
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.746179628372192,
      "eval_nll_ablated": 2.746179628372192,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.582985225900506,
      "perplexity_per_byte_ablated": 15.582985225900506
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.633123763402303,
      "eval_nll_ablated": 2.633123763402303,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.917176036753492,
      "perplexity_per_byte_ablated": 13.917176036753492
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5355427185694377,
      "eval_nll_ablated": 2.5355427185694377,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.62327987771784,
      "perplexity_per_byte_ablated": 12.62327987771784
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.368552494049072,
      "eval_nll_ablated": 2.368552494049072,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.681918946696126,
      "perplexity_per_byte_ablated": 10.681918946696126
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.745886778831482,
      "eval_nll_ablated": 2.7530340274175007,
      "label_use_gap": 0.007147248586018673,
      "perplexity_per_byte_labeled": 15.578422423974041,
      "perplexity_per_byte_ablated": 15.690164128810906
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6323579947153726,
      "eval_nll_ablated": 2.635241897900899,
      "label_use_gap": 0.0028839031855265596,
      "perplexity_per_byte_labeled": 13.906522778620385,
      "perplexity_per_byte_ablated": 13.946685729155815
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5402905384699506,
      "eval_nll_ablated": 2.5445667187372845,
      "label_use_gap": 0.0042761802673338956,
      "perplexity_per_byte_labeled": 12.683355438251064,
      "perplexity_per_byte_ablated": 12.737707880096588
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3815883080164593,
      "eval_nll_ablated": 2.3936396201451617,
      "label_use_gap": 0.012051312128702474,
      "perplexity_per_byte_labeled": 10.822078013870593,
      "perplexity_per_byte_ablated": 10.9532872878529
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7466595649719237,
      "eval_nll_ablated": 2.7520291805267334,
      "label_use_gap": 0.005369615554809748,
      "perplexity_per_byte_labeled": 15.590465865816341,
      "perplexity_per_byte_ablated": 15.674405834832747
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6344762643178306,
      "eval_nll_ablated": 2.6381080706914264,
      "label_use_gap": 0.003631806373595836,
      "perplexity_per_byte_labeled": 13.936011764883244,
      "perplexity_per_byte_ablated": 13.98671668071849
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5433234373728433,
      "eval_nll_ablated": 2.545336922009786,
      "label_use_gap": 0.0020134846369428416,
      "perplexity_per_byte_labeled": 12.721881165831697,
      "perplexity_per_byte_ablated": 12.747522283447628
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3778982639312742,
      "eval_nll_ablated": 2.3888302008310953,
      "label_use_gap": 0.010931936899821082,
      "perplexity_per_byte_labeled": 10.782217657372085,
      "perplexity_per_byte_ablated": 10.900734811163488
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7413500626881917,
      "eval_nll_ablated": 2.7459752798080443,
      "label_use_gap": 0.004625217119852643,
      "perplexity_per_byte_labeled": 15.507907617250593,
      "perplexity_per_byte_ablated": 15.579801190582058
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.632516074180603,
      "eval_nll_ablated": 2.6372051159540812,
      "label_use_gap": 0.004689041773478397,
      "perplexity_per_byte_labeled": 13.908721288069462,
      "perplexity_per_byte_ablated": 13.974093008791618
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5421513954798383,
      "eval_nll_ablated": 2.5451953252156576,
      "label_use_gap": 0.0030439297358193507,
      "perplexity_per_byte_labeled": 12.706979322645626,
      "perplexity_per_byte_ablated": 12.745717402944871
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.378053331375122,
      "eval_nll_ablated": 2.3903494675954184,
      "label_use_gap": 0.01229613622029646,
      "perplexity_per_byte_labeled": 10.783889757944053,
      "perplexity_per_byte_ablated": 10.917308522024912
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7416547377904257,
      "eval_nll_ablated": 2.748112098375956,
      "label_use_gap": 0.0064573605855304805,
      "perplexity_per_byte_labeled": 15.512633210438054,
      "perplexity_per_byte_ablated": 15.613127993034784
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.63221218585968,
      "eval_nll_ablated": 2.636231271425883,
      "label_use_gap": 0.004019085566203007,
      "perplexity_per_byte_labeled": 13.90449523226807,
      "perplexity_per_byte_ablated": 13.960491038955402
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.544094475110372,
      "eval_nll_ablated": 2.547045342127482,
      "label_use_gap": 0.002950867017110337,
      "perplexity_per_byte_labeled": 12.731693998849048,
      "perplexity_per_byte_ablated": 12.769319020658113
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3837454160054525,
      "eval_nll_ablated": 2.395306285222371,
      "label_use_gap": 0.011560869216918679,
      "perplexity_per_byte_labeled": 10.84544760111171,
      "perplexity_per_byte_ablated": 10.97155797058131
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7415406306584678,
      "eval_nll_ablated": 2.747736914952596,
      "label_use_gap": 0.006196284294128329,
      "perplexity_per_byte_labeled": 15.510863209340089,
      "perplexity_per_byte_ablated": 15.60727130495984
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6325397650400797,
      "eval_nll_ablated": 2.6379581610361735,
      "label_use_gap": 0.005418395996093839,
      "perplexity_per_byte_labeled": 13.909050801534214,
      "perplexity_per_byte_ablated": 13.984620093995987
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.541234763463338,
      "eval_nll_ablated": 2.547519000371297,
      "label_use_gap": 0.006284236907958984,
      "perplexity_per_byte_labeled": 12.695337035227347,
      "perplexity_per_byte_ablated": 12.775368746518309
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3784207344055175,
      "eval_nll_ablated": 2.394588343302409,
      "label_use_gap": 0.016167608896891394,
      "perplexity_per_byte_labeled": 10.787852519641325,
      "perplexity_per_byte_ablated": 10.963683856103586
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.745737822850545,
      "eval_nll_ablated": 2.753430485725403,
      "label_use_gap": 0.007692662874857703,
      "perplexity_per_byte_labeled": 15.576102097597964,
      "perplexity_per_byte_ablated": 15.696385857978736
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6336037000020345,
      "eval_nll_ablated": 2.6365939458211263,
      "label_use_gap": 0.002990245819091797,
      "perplexity_per_byte_labeled": 13.923857001990065,
      "perplexity_per_byte_ablated": 13.965555069841916
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5407280206680296,
      "eval_nll_ablated": 2.5441240231196085,
      "label_use_gap": 0.0033960024515788945,
      "perplexity_per_byte_labeled": 12.688905394382195,
      "perplexity_per_byte_ablated": 12.732070200618903
    },
    {
      "train_per_category": 100,
      "seed": 25219,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3785236358642576,
      "eval_nll_ablated": 2.3890470345815022,
      "label_use_gap": 0.010523398717244614,
      "perplexity_per_byte_labeled": 10.788962662518951,
      "perplexity_per_byte_ablated": 10.903098714652552
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7268791755040485,
      "eval_nll_ablated": 2.7268791755040485,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.285110360563126,
      "perplexity_per_byte_ablated": 15.285110360563126
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6326179265975953,
      "eval_nll_ablated": 2.6326179265975953,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.91013799709632,
      "perplexity_per_byte_ablated": 13.91013799709632
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5480041027069094,
      "eval_nll_ablated": 2.5480041027069094,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.781567611155847,
      "perplexity_per_byte_ablated": 12.781567611155847
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3795338312784833,
      "eval_nll_ablated": 2.3795338312784833,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.799867130019082,
      "perplexity_per_byte_ablated": 10.799867130019082
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7254855314890545,
      "eval_nll_ablated": 2.727805256843567,
      "label_use_gap": 0.002319725354512414,
      "perplexity_per_byte_labeled": 15.26382319480159,
      "perplexity_per_byte_ablated": 15.29927217252329
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.632184211413066,
      "eval_nll_ablated": 2.634568436940511,
      "label_use_gap": 0.0023842255274453272,
      "perplexity_per_byte_labeled": 13.904106267149064,
      "perplexity_per_byte_ablated": 13.937296342837872
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5341890335083006,
      "eval_nll_ablated": 2.54514209429423,
      "label_use_gap": 0.010953060785929392,
      "perplexity_per_byte_labeled": 12.60620349295595,
      "perplexity_per_byte_ablated": 12.745038954720629
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.384486182530721,
      "eval_nll_ablated": 2.400090138117472,
      "label_use_gap": 0.015603955586751095,
      "perplexity_per_byte_labeled": 10.853484522019643,
      "perplexity_per_byte_ablated": 11.024170033791457
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7265664259592692,
      "eval_nll_ablated": 2.7286779642105103,
      "label_use_gap": 0.0021115382512411074,
      "perplexity_per_byte_labeled": 15.280330696713765,
      "perplexity_per_byte_ablated": 15.312629787853991
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6317286014556887,
      "eval_nll_ablated": 2.6334160884221394,
      "label_use_gap": 0.001687486966450713,
      "perplexity_per_byte_labeled": 13.897772860776383,
      "perplexity_per_byte_ablated": 13.921244970211053
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5316057205200195,
      "eval_nll_ablated": 2.5408079783121744,
      "label_use_gap": 0.009202257792154889,
      "perplexity_per_byte_labeled": 12.573679751329049,
      "perplexity_per_byte_ablated": 12.6899200099269
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.380476379394531,
      "eval_nll_ablated": 2.3924605290095013,
      "label_use_gap": 0.011984149614970185,
      "perplexity_per_byte_labeled": 10.810051323228143,
      "perplexity_per_byte_ablated": 10.940379974849996
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7266539414723714,
      "eval_nll_ablated": 2.7291630109151206,
      "label_use_gap": 0.002509069442749201,
      "perplexity_per_byte_labeled": 15.281668021212525,
      "perplexity_per_byte_ablated": 15.320058930066326
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.632121435801188,
      "eval_nll_ablated": 2.6337608178456624,
      "label_use_gap": 0.0016393820444742246,
      "perplexity_per_byte_labeled": 13.903233455766454,
      "perplexity_per_byte_ablated": 13.926044860248485
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.534357754389445,
      "eval_nll_ablated": 2.5424710432688395,
      "label_use_gap": 0.008113288879394442,
      "perplexity_per_byte_labeled": 12.608330602155991,
      "perplexity_per_byte_ablated": 12.711041729726125
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.380004342397054,
      "eval_nll_ablated": 2.393421681722005,
      "label_use_gap": 0.01341733932495126,
      "perplexity_per_byte_labeled": 10.804949783211493,
      "perplexity_per_byte_ablated": 10.950900405799105
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.72649093468984,
      "eval_nll_ablated": 2.7285924514134723,
      "label_use_gap": 0.0021015167236324572,
      "perplexity_per_byte_labeled": 15.279177208691854,
      "perplexity_per_byte_ablated": 15.31132041803556
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.631518769264221,
      "eval_nll_ablated": 2.634039799372355,
      "label_use_gap": 0.0025210301081339814,
      "perplexity_per_byte_labeled": 13.894856966575421,
      "perplexity_per_byte_ablated": 13.929930511491644
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.532508691151937,
      "eval_nll_ablated": 2.5423104842503865,
      "label_use_gap": 0.00980179309844953,
      "perplexity_per_byte_labeled": 12.585038542435237,
      "perplexity_per_byte_ablated": 12.70900102117395
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3799858570098875,
      "eval_nll_ablated": 2.3949633916219075,
      "label_use_gap": 0.014977534612020005,
      "perplexity_per_byte_labeled": 10.804750051377503,
      "perplexity_per_byte_ablated": 10.967796538488432
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7269439776738484,
      "eval_nll_ablated": 2.7290420850118,
      "label_use_gap": 0.0020981073379515713,
      "perplexity_per_byte_labeled": 15.28610090097436,
      "perplexity_per_byte_ablated": 15.318206450109948
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6329002141952516,
      "eval_nll_ablated": 2.6335071325302124,
      "label_use_gap": 0.0006069183349608487,
      "perplexity_per_byte_labeled": 13.914065210810373,
      "perplexity_per_byte_ablated": 13.92251247524119
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5365132490793862,
      "eval_nll_ablated": 2.544391171137492,
      "label_use_gap": 0.007877922058105558,
      "perplexity_per_byte_labeled": 12.635537103018226,
      "perplexity_per_byte_ablated": 12.735472002308608
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3818930625915526,
      "eval_nll_ablated": 2.3957521518071494,
      "label_use_gap": 0.013859089215596754,
      "perplexity_per_byte_labeled": 10.82537659426054,
      "perplexity_per_byte_ablated": 10.976450912381916
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7273468097050984,
      "eval_nll_ablated": 2.7292359749476116,
      "label_use_gap": 0.0018891652425132577,
      "perplexity_per_byte_labeled": 15.292259872482406,
      "perplexity_per_byte_ablated": 15.321176784124939
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6328250646591185,
      "eval_nll_ablated": 2.634327451388041,
      "label_use_gap": 0.0015023867289225556,
      "perplexity_per_byte_labeled": 13.913019614552583,
      "perplexity_per_byte_ablated": 13.933938060443774
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.53573583761851,
      "eval_nll_ablated": 2.545325287183126,
      "label_use_gap": 0.009589449564615826,
      "perplexity_per_byte_labeled": 12.625717908931556,
      "perplexity_per_byte_ablated": 12.747373969098318
    },
    {
      "train_per_category": 100,
      "seed": 26227,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3796708663304647,
      "eval_nll_ablated": 2.39584219455719,
      "label_use_gap": 0.01617132822672529,
      "perplexity_per_byte_labeled": 10.80134719178049,
      "perplexity_per_byte_ablated": 10.977439306705955
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "none",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.736895489692688,
      "eval_nll_ablated": 2.736895489692688,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.438980145966681,
      "perplexity_per_byte_ablated": 15.438980145966681
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6337908188501995,
      "eval_nll_ablated": 2.6337908188501995,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.926462661850724,
      "perplexity_per_byte_ablated": 13.926462661850724
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.541372052828471,
      "eval_nll_ablated": 2.541372052828471,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.69708008963774,
      "perplexity_per_byte_ablated": 12.69708008963774
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3620183865229287,
      "eval_nll_ablated": 2.3620183865229287,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 10.612349673786673,
      "perplexity_per_byte_ablated": 10.612349673786673
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "hierarchy",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7385587056477863,
      "eval_nll_ablated": 2.7400983730951944,
      "label_use_gap": 0.0015396674474081884,
      "perplexity_per_byte_labeled": 15.464679870247233,
      "perplexity_per_byte_ablated": 15.488508673937385
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.634869114557902,
      "eval_nll_ablated": 2.63721714814504,
      "label_use_gap": 0.0023480335871379765,
      "perplexity_per_byte_labeled": 13.941487605973036,
      "perplexity_per_byte_ablated": 13.974261148758718
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.537015748023987,
      "eval_nll_ablated": 2.5373201926549274,
      "label_use_gap": 0.00030444463094037033,
      "perplexity_per_byte_labeled": 12.641888042613541,
      "perplexity_per_byte_ablated": 12.645737383478911
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.359568126996358,
      "eval_nll_ablated": 2.365390833218892,
      "label_use_gap": 0.005822706222534002,
      "perplexity_per_byte_labeled": 10.586378493958327,
      "perplexity_per_byte_ablated": 10.648199674510673
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_1",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7380596160888673,
      "eval_nll_ablated": 2.7395121653874717,
      "label_use_gap": 0.0014525492986043886,
      "perplexity_per_byte_labeled": 15.456963535723137,
      "perplexity_per_byte_ablated": 15.479431851483163
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.637195873260498,
      "eval_nll_ablated": 2.639389252662659,
      "label_use_gap": 0.002193379402160911,
      "perplexity_per_byte_labeled": 13.973963851128715,
      "perplexity_per_byte_ablated": 14.004647693959303
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5404229879379274,
      "eval_nll_ablated": 2.539459164937337,
      "label_use_gap": -0.0009638230005903914,
      "perplexity_per_byte_labeled": 12.685035453187108,
      "perplexity_per_byte_ablated": 12.672815214273694
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.362951668103536,
      "eval_nll_ablated": 2.366072185834249,
      "label_use_gap": 0.0031205177307129794,
      "perplexity_per_byte_labeled": 10.622258607457583,
      "perplexity_per_byte_ablated": 10.655457325436686
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_2",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.737804325421651,
      "eval_nll_ablated": 2.740045913060506,
      "label_use_gap": 0.0022415876388550693,
      "perplexity_per_byte_labeled": 15.453018020836959,
      "perplexity_per_byte_ablated": 15.487696167547323
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6363588174184165,
      "eval_nll_ablated": 2.638781189918518,
      "label_use_gap": 0.0024223725001015772,
      "perplexity_per_byte_labeled": 13.962271757200542,
      "perplexity_per_byte_ablated": 13.996134577968428
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5379895051320394,
      "eval_nll_ablated": 2.5379861275355022,
      "label_use_gap": -3.3775965371241057e-06,
      "perplexity_per_byte_labeled": 12.654204166437614,
      "perplexity_per_byte_ablated": 12.654161425713623
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3640899578730266,
      "eval_nll_ablated": 2.367124597231547,
      "label_use_gap": 0.00303463935852033,
      "perplexity_per_byte_labeled": 10.63435670002054,
      "perplexity_per_byte_ablated": 10.666677153071076
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_3",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7376355568567914,
      "eval_nll_ablated": 2.740115253130595,
      "label_use_gap": 0.002479696273803622,
      "perplexity_per_byte_labeled": 15.45041025722328,
      "perplexity_per_byte_ablated": 15.48877012271873
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.636367122332255,
      "eval_nll_ablated": 2.638397208849589,
      "label_use_gap": 0.002030086517334162,
      "perplexity_per_byte_labeled": 13.962387713145976,
      "perplexity_per_byte_ablated": 13.990761358925521
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5380701541900637,
      "eval_nll_ablated": 2.5377283573150633,
      "label_use_gap": -0.00034179687500035527,
      "perplexity_per_byte_labeled": 12.655224757237974,
      "perplexity_per_byte_ablated": 12.650899980103283
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.36135090192159,
      "eval_nll_ablated": 2.3676034291585286,
      "label_use_gap": 0.0062525272369384766,
      "perplexity_per_byte_labeled": 10.605268457359772,
      "perplexity_per_byte_ablated": 10.671785921669917
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_4",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.7382699330647786,
      "eval_nll_ablated": 2.740025504430135,
      "label_use_gap": 0.0017555713653565341,
      "perplexity_per_byte_labeled": 15.460214739430423,
      "perplexity_per_byte_ablated": 15.487380088106327
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.634612520535787,
      "eval_nll_ablated": 2.636619289716085,
      "label_use_gap": 0.0020067691802978516,
      "perplexity_per_byte_labeled": 13.937910762511905,
      "perplexity_per_byte_ablated": 13.965909015887812
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.539082320531209,
      "eval_nll_ablated": 2.5378501494725545,
      "label_use_gap": -0.0012321710586546963,
      "perplexity_per_byte_labeled": 12.668040434481338,
      "perplexity_per_byte_ablated": 12.652440854337359
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.3611550172170004,
      "eval_nll_ablated": 2.364327883720398,
      "label_use_gap": 0.0031728665033976533,
      "perplexity_per_byte_labeled": 10.603191250934033,
      "perplexity_per_byte_ablated": 10.636887189372118
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_5",
      "step": 100,
      "examples_processed": 2400,
      "eval_nll_labeled": 2.738814695676168,
      "eval_nll_ablated": 2.7404165665308633,
      "label_use_gap": 0.0016018708546954308,
      "perplexity_per_byte_labeled": 15.468639180836524,
      "perplexity_per_byte_ablated": 15.493437799891906
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.6353955904642743,
      "eval_nll_ablated": 2.6375547647476196,
      "label_use_gap": 0.002159174283345333,
      "perplexity_per_byte_labeled": 13.948829395764704,
      "perplexity_per_byte_ablated": 13.978979887848832
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.5367612202962238,
      "eval_nll_ablated": 2.5370861689249673,
      "label_use_gap": 0.0003249486287435488,
      "perplexity_per_byte_labeled": 12.638670741039016,
      "perplexity_per_byte_ablated": 12.642778327106544
    },
    {
      "train_per_category": 100,
      "seed": 27239,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.360185488065084,
      "eval_nll_ablated": 2.364784264564514,
      "label_use_gap": 0.0045987764994301905,
      "perplexity_per_byte_labeled": 10.592916129732552,
      "perplexity_per_byte_ablated": 10.641742768838819
    }
  ]
}