{
  "experiment": "H1 real-text conditional byte-model pilot v7 (several scrambled permutations)",
  "completed_utc": "2026-11-18T01:26:00.978582+00:00",
  "smoke_test_not_protocol": false,
  "protocol": {
    "dataset": "SetFit/20_newsgroups",
    "revision": "f1b91292074e7cfb69be58b642d583ec262f30ed",
    "dataset_split": "train only; official test unused",
    "selected_data_sha256": "72325b82d18ed20a73b97086b6447e027c820a42c3ce4440cf75e384875095c1",
    "categories": [
      "comp.graphics",
      "comp.os.ms-windows.misc",
      "comp.sys.ibm.pc.hardware",
      "comp.sys.mac.hardware",
      "rec.autos",
      "rec.motorcycles",
      "rec.sport.baseball",
      "rec.sport.hockey"
    ],
    "train_per_category": 300,
    "validation_per_category": 80,
    "validation_messages": 640,
    "bytes_per_example": 96,
    "seed_values": [
      2027,
      3037,
      4049,
      5051,
      6067
    ],
    "steps": [
      200,
      400,
      800
    ],
    "total_updates": 800,
    "batch_size": 24,
    "conditions": [
      "none",
      "hierarchy",
      "scrambled_1",
      "scrambled_2",
      "scrambled_3",
      "scrambled_4",
      "scrambled_5"
    ],
    "permutation_seeds_selected": [
      31415,
      27182,
      16180,
      14142,
      17320
    ],
    "permutation_seeds_rejected": [],
    "permutation_seeds_primary": [
      31415,
      27182,
      16180,
      14142,
      17320
    ],
    "permutation_seeds_fallback": [
      11235,
      13579,
      24680,
      97531
    ],
    "scrambled_permutations": {
      "scrambled_1": [
        2,
        1,
        5,
        6,
        7,
        0,
        3,
        4
      ],
      "scrambled_2": [
        1,
        2,
        5,
        6,
        7,
        3,
        0,
        4
      ],
      "scrambled_3": [
        5,
        3,
        0,
        2,
        4,
        7,
        6,
        1
      ],
      "scrambled_4": [
        5,
        3,
        4,
        0,
        6,
        2,
        7,
        1
      ],
      "scrambled_5": [
        1,
        6,
        4,
        5,
        2,
        7,
        0,
        3
      ]
    },
    "permutation_rejection_rule": "rejected if categories 0-3 all fall in the same code//4 group (comp/rec split preserved)",
    "same_batch_sequence_per_seed_across_conditions": true,
    "evaluations": "each checkpoint: (a) eval_nll_labeled with the condition label prefix; (b) eval_nll_ablated with dummy prefix [256, 256] for all conditions; label_use_gap = ablated - labeled",
    "metric": "validation NLL in nats per byte, lower is better",
    "model": "randomly initialized 2-layer transformer width 64, 4 heads, FF 128",
    "limitations": "five seeds, five scrambled permutations, only two groups of four categories, short prefixes, tiny model (800 updates), one historical text dataset (20 Newsgroups); no significance test is run or claimed; no data-savings percentage is estimated; labels present at labeled evaluation; a null result is informative; not a pretrained-LLM or general data-savings result"
  },
  "environment": {
    "device": "cuda",
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition MIG 2g.48gb",
    "torch": "2.13.0+cu130",
    "cuda": "13.0",
    "python": "3.12.12",
    "code_sha256": "5e27b5abda486f63dcb9dd0d7e0c9a85d739f88a9bdacb5f7a6c633a37885c9c"
  },
  "summary": {
    "200": {
      "mean_eval_nll_labeled_over_seeds": {
        "none": 2.7292309745152794,
        "hierarchy": 2.732157718340556,
        "scrambled_1": 2.731982259750366,
        "scrambled_2": 2.7324018891652426,
        "scrambled_3": 2.7322260840733845,
        "scrambled_4": 2.732134267489115,
        "scrambled_5": 2.7322136147816978,
        "scrambled_mean": 2.7321916230519614
      },
      "paired_diff_hierarchy_minus_scrambled_mean": {
        "mean": -3.390471140560791e-05,
        "min": -0.000614094734192161,
        "max": 0.000652238527933946,
        "per_seed": {
          "2027": -4.00352478031607e-05,
          "3037": -0.0003744029998777876,
          "4049": 0.000652238527933946,
          "5051": 0.00020677089691112371,
          "6067": -0.000614094734192161
        }
      },
      "n_scrambled_better_than_hierarchy": {
        "mean_over_seeds": 2.2,
        "per_seed": {
          "2027": 2,
          "3037": 0,
          "4049": 5,
          "5051": 3,
          "6067": 1
        }
      },
      "hierarchy_rank_among_6_codes": {
        "mean_over_seeds": 3.2,
        "per_seed": {
          "2027": 3,
          "3037": 1,
          "4049": 6,
          "5051": 4,
          "6067": 2
        }
      },
      "between_permutation_range_of_scrambled": {
        "mean_over_seeds": 0.0011023569107056375,
        "per_seed": {
          "2027": 0.0015781005223591826,
          "3037": 0.00023125807444257518,
          "4049": 0.0009472688039142696,
          "5051": 0.0014965852101647137,
          "6067": 0.0012585719426474462
        }
      },
      "seeds_where_hierarchy_beats_scrambled_mean": 3
    },
    "400": {
      "mean_eval_nll_labeled_over_seeds": {
        "none": 2.613428896268209,
        "hierarchy": 2.616026469866435,
        "scrambled_1": 2.615664536158244,
        "scrambled_2": 2.6158019622166955,
        "scrambled_3": 2.6161598714192706,
        "scrambled_4": 2.616745583216349,
        "scrambled_5": 2.6166405296325683,
        "scrambled_mean": 2.6162024965286252
      },
      "paired_diff_hierarchy_minus_scrambled_mean": {
        "mean": -0.00017602666219067585,
        "min": -0.0022230927149453628,
        "max": 0.003294556935628634,
        "per_seed": {
          "2027": 0.003294556935628634,
          "3037": -0.001577690442402968,
          "4049": 5.7800610859803925e-05,
          "5051": -0.0022230927149453628,
          "6067": -0.0004317077000934866
        }
      },
      "n_scrambled_better_than_hierarchy": {
        "mean_over_seeds": 1.4,
        "per_seed": {
          "2027": 5,
          "3037": 0,
          "4049": 2,
          "5051": 0,
          "6067": 0
        }
      },
      "hierarchy_rank_among_6_codes": {
        "mean_over_seeds": 2.4,
        "per_seed": {
          "2027": 6,
          "3037": 1,
          "4049": 3,
          "5051": 1,
          "6067": 1
        }
      },
      "between_permutation_range_of_scrambled": {
        "mean_over_seeds": 0.0018607902526855292,
        "per_seed": {
          "2027": 0.002387666702270863,
          "3037": 0.002009797096252708,
          "4049": 0.002168734868367217,
          "5051": 0.0021168708801266867,
          "6067": 0.000620881716410171
        }
      },
      "seeds_where_hierarchy_beats_scrambled_mean": 3
    },
    "800": {
      "mean_eval_nll_labeled_over_seeds": {
        "none": 2.5322701756159463,
        "hierarchy": 2.5278536987304685,
        "scrambled_1": 2.5286077149709065,
        "scrambled_2": 2.5278021701176963,
        "scrambled_3": 2.5295955880482994,
        "scrambled_4": 2.5302836338679,
        "scrambled_5": 2.531011160214742,
        "scrambled_mean": 2.5294600534439087
      },
      "paired_diff_hierarchy_minus_scrambled_mean": {
        "mean": -0.0016063547134400834,
        "min": -0.004410661061604593,
        "max": 0.0028062343597410333,
        "per_seed": {
          "2027": -0.00030887285868352876,
          "3037": -0.0038047583897911075,
          "4049": 0.0028062343597410333,
          "5051": -0.004410661061604593,
          "6067": -0.002313715616862222
        }
      },
      "n_scrambled_better_than_hierarchy": {
        "mean_over_seeds": 1.6,
        "per_seed": {
          "2027": 2,
          "3037": 0,
          "4049": 5,
          "5051": 0,
          "6067": 1
        }
      },
      "hierarchy_rank_among_6_codes": {
        "mean_over_seeds": 2.6,
        "per_seed": {
          "2027": 3,
          "3037": 1,
          "4049": 6,
          "5051": 1,
          "6067": 2
        }
      },
      "between_permutation_range_of_scrambled": {
        "mean_over_seeds": 0.005215234756469567,
        "per_seed": {
          "2027": 0.003401144345601015,
          "3037": 0.007804902394612512,
          "4049": 0.004741295178731342,
          "5051": 0.0035619020462034356,
          "6067": 0.006566929817199529
        }
      },
      "seeds_where_hierarchy_beats_scrambled_mean": 4
    }
  },
  "elapsed_seconds": 111.37210081199919,
  "runs": [
    {
      "seed": 2027,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.713966719309489,
      "eval_nll_ablated": 2.713966719309489,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.089010830555747,
      "perplexity_per_byte_ablated": 15.089010830555747
    },
    {
      "seed": 2027,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6099636713663736,
      "eval_nll_ablated": 2.6099636713663736,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.598556825868561,
      "perplexity_per_byte_ablated": 13.598556825868561
    },
    {
      "seed": 2027,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5254138708114624,
      "eval_nll_ablated": 2.5254138708114624,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.49606595048983,
      "perplexity_per_byte_ablated": 12.49606595048983
    },
    {
      "seed": 2027,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.723436156908671,
      "eval_nll_ablated": 2.7338385264078777,
      "label_use_gap": 0.01040236949920681,
      "perplexity_per_byte_labeled": 15.232573935202907,
      "perplexity_per_byte_ablated": 15.391855815877165
    },
    {
      "seed": 2027,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6133323510487876,
      "eval_nll_ablated": 2.6390655994415284,
      "label_use_gap": 0.025733248392740826,
      "perplexity_per_byte_labeled": 13.644443252901972,
      "perplexity_per_byte_ablated": 14.000115778046508
    },
    {
      "seed": 2027,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.509923982620239,
      "eval_nll_ablated": 2.5405718008677165,
      "label_use_gap": 0.03064781824747742,
      "perplexity_per_byte_labeled": 12.303994707521062,
      "perplexity_per_byte_ablated": 12.686923290941225
    },
    {
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7230921268463133,
      "eval_nll_ablated": 2.7361203114191692,
      "label_use_gap": 0.013028184572855928,
      "perplexity_per_byte_labeled": 15.227334373177214,
      "perplexity_per_byte_ablated": 15.427016821445932
    },
    {
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.611062264442444,
      "eval_nll_ablated": 2.638297112782796,
      "label_use_gap": 0.02723484834035217,
      "perplexity_per_byte_labeled": 13.613504315342878,
      "perplexity_per_byte_ablated": 13.989361008827991
    },
    {
      "seed": 2027,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5099265575408936,
      "eval_nll_ablated": 2.549450079600016,
      "label_use_gap": 0.03952352205912257,
      "perplexity_per_byte_labeled": 12.304026389371957,
      "perplexity_per_byte_ablated": 12.8000628311871
    },
    {
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7237094163894655,
      "eval_nll_ablated": 2.735919300715129,
      "label_use_gap": 0.012209884325663278,
      "perplexity_per_byte_labeled": 15.236736949213224,
      "perplexity_per_byte_ablated": 15.423916137579173
    },
    {
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6089345932006838,
      "eval_nll_ablated": 2.6343914270401,
      "label_use_gap": 0.025456833839416326,
      "perplexity_per_byte_labeled": 13.584570045933424,
      "perplexity_per_byte_ablated": 13.934829521732546
    },
    {
      "seed": 2027,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5096522331237794,
      "eval_nll_ablated": 2.5435274998346964,
      "label_use_gap": 0.03387526671091701,
      "perplexity_per_byte_labeled": 12.300651557425102,
      "perplexity_per_byte_ablated": 12.72447748911885
    },
    {
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7227489868799846,
      "eval_nll_ablated": 2.7340996583302815,
      "label_use_gap": 0.01135067145029689,
      "perplexity_per_byte_labeled": 15.222110162542117,
      "perplexity_per_byte_ablated": 15.395875645605816
    },
    {
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.608829061190287,
      "eval_nll_ablated": 2.6325385649998982,
      "label_use_gap": 0.02370950380961112,
      "perplexity_per_byte_labeled": 13.583136514589155,
      "perplexity_per_byte_ablated": 13.90903411012438
    },
    {
      "seed": 2027,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.510261797904968,
      "eval_nll_ablated": 2.533938201268514,
      "label_use_gap": 0.023676403363545706,
      "perplexity_per_byte_labeled": 12.308151887136361,
      "perplexity_per_byte_ablated": 12.60304184723652
    },
    {
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.723503343264262,
      "eval_nll_ablated": 2.734459074338277,
      "label_use_gap": 0.010955731074015151,
      "perplexity_per_byte_labeled": 15.233597390712621,
      "perplexity_per_byte_ablated": 15.401410164307686
    },
    {
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6101463238398233,
      "eval_nll_ablated": 2.633265463511149,
      "label_use_gap": 0.02311913967132595,
      "perplexity_per_byte_labeled": 13.601040862758987,
      "perplexity_per_byte_ablated": 13.919148241840269
    },
    {
      "seed": 2027,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5089612722396852,
      "eval_nll_ablated": 2.5376033544540406,
      "label_use_gap": 0.02864208221435538,
      "perplexity_per_byte_labeled": 12.292155224005107,
      "perplexity_per_byte_ablated": 12.64931868024682
    },
    {
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7243270874023438,
      "eval_nll_ablated": 2.7353185494740804,
      "label_use_gap": 0.010991462071736624,
      "perplexity_per_byte_labeled": 15.24615114709687,
      "perplexity_per_byte_ablated": 15.414652983521934
    },
    {
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.611216727892558,
      "eval_nll_ablated": 2.637114135424296,
      "label_use_gap": 0.025897407531738192,
      "perplexity_per_byte_labeled": 13.61560726659794,
      "perplexity_per_byte_ablated": 13.972821696239633
    },
    {
      "seed": 2027,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5123624165852863,
      "eval_nll_ablated": 2.545703132947286,
      "label_use_gap": 0.03334071636199987,
      "perplexity_per_byte_labeled": 12.334033795403462,
      "perplexity_per_byte_ablated": 12.752191420426184
    },
    {
      "seed": 3037,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7246466795603435,
      "eval_nll_ablated": 2.7246466795603435,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.251024476140552,
      "perplexity_per_byte_ablated": 15.251024476140552
    },
    {
      "seed": 3037,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6137926816940307,
      "eval_nll_ablated": 2.6137926816940307,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.650725654148534,
      "perplexity_per_byte_ablated": 13.650725654148534
    },
    {
      "seed": 3037,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5286266724268596,
      "eval_nll_ablated": 2.5286266724268596,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.536277893269752,
      "perplexity_per_byte_ablated": 12.536277893269752
    },
    {
      "seed": 3037,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.727604349454244,
      "eval_nll_ablated": 2.733096655209859,
      "label_use_gap": 0.005492305755614879,
      "perplexity_per_byte_labeled": 15.296198744440114,
      "perplexity_per_byte_ablated": 15.380441275946037
    },
    {
      "seed": 3037,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.616090973218282,
      "eval_nll_ablated": 2.6227081934611003,
      "label_use_gap": 0.006617220242818078,
      "perplexity_per_byte_labeled": 13.68213508140381,
      "perplexity_per_byte_ablated": 13.772972998416765
    },
    {
      "seed": 3037,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.530032761891683,
      "eval_nll_ablated": 2.540868886311849,
      "label_use_gap": 0.010836124420166016,
      "perplexity_per_byte_labeled": 12.553917420013695,
      "perplexity_per_byte_ablated": 12.690692951109645
    },
    {
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7280321518580117,
      "eval_nll_ablated": 2.733606386184692,
      "label_use_gap": 0.005574234326680472,
      "perplexity_per_byte_labeled": 15.30274389494723,
      "perplexity_per_byte_ablated": 15.388283161727237
    },
    {
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6165419181187946,
      "eval_nll_ablated": 2.6246172904968263,
      "label_use_gap": 0.008075372378031798,
      "perplexity_per_byte_labeled": 13.688306361796023,
      "perplexity_per_byte_ablated": 13.79929205516414
    },
    {
      "seed": 3037,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.530243484179179,
      "eval_nll_ablated": 2.5470990419387816,
      "label_use_gap": 0.01685555775960257,
      "perplexity_per_byte_labeled": 12.556563088950393,
      "perplexity_per_byte_ablated": 12.770004749091523
    },
    {
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.728016193707784,
      "eval_nll_ablated": 2.734409697850545,
      "label_use_gap": 0.006393504142761408,
      "perplexity_per_byte_labeled": 15.302499693409763,
      "perplexity_per_byte_ablated": 15.400649715541954
    },
    {
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.616862487792969,
      "eval_nll_ablated": 2.6262913862864177,
      "label_use_gap": 0.009428898493448745,
      "perplexity_per_byte_labeled": 13.692695121120412,
      "perplexity_per_byte_ablated": 13.822412739613446
    },
    {
      "seed": 3037,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.530332390467326,
      "eval_nll_ablated": 2.5449832518895468,
      "label_use_gap": 0.014650861422220718,
      "perplexity_per_byte_labeled": 12.557679495993584,
      "perplexity_per_byte_ablated": 12.74301466286123
    },
    {
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.727828621864319,
      "eval_nll_ablated": 2.734055924415588,
      "label_use_gap": 0.006227302551269354,
      "perplexity_per_byte_labeled": 15.299629644511244,
      "perplexity_per_byte_ablated": 15.39520233841699
    },
    {
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6180112600326537,
      "eval_nll_ablated": 2.624495808283488,
      "label_use_gap": 0.006484548250834443,
      "perplexity_per_byte_labeled": 13.708433947594644,
      "perplexity_per_byte_ablated": 13.797615788443135
    },
    {
      "seed": 3037,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5380483865737915,
      "eval_nll_ablated": 2.544976735115051,
      "label_use_gap": 0.006928348541259677,
      "perplexity_per_byte_labeled": 12.654949286159807,
      "perplexity_per_byte_ablated": 12.742931619778865
    },
    {
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7279569149017333,
      "eval_nll_ablated": 2.734509007136027,
      "label_use_gap": 0.006552092234293649,
      "perplexity_per_byte_labeled": 15.301592606384133,
      "perplexity_per_byte_ablated": 15.40217921900685
    },
    {
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6185517152150473,
      "eval_nll_ablated": 2.6253050486246745,
      "label_use_gap": 0.006753333409627249,
      "perplexity_per_byte_labeled": 13.715844744185446,
      "perplexity_per_byte_ablated": 13.80878589479209
    },
    {
      "seed": 3037,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5374178806940715,
      "eval_nll_ablated": 2.544642702738444,
      "label_use_gap": 0.00722482204437247,
      "perplexity_per_byte_labeled": 12.646972781108238,
      "perplexity_per_byte_ablated": 12.738675778878862
    },
    {
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7280598799387614,
      "eval_nll_ablated": 2.7339389006296795,
      "label_use_gap": 0.005879020690918058,
      "perplexity_per_byte_labeled": 15.303168216548427,
      "perplexity_per_byte_ablated": 15.393400838965716
    },
    {
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6183759371439614,
      "eval_nll_ablated": 2.624864029884338,
      "label_use_gap": 0.006488092740376761,
      "perplexity_per_byte_labeled": 13.713434011336194,
      "perplexity_per_byte_ablated": 13.802697304121173
    },
    {
      "seed": 3037,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.533145459493001,
      "eval_nll_ablated": 2.542188533147176,
      "label_use_gap": 0.009043073654174894,
      "perplexity_per_byte_labeled": 12.593054848544238,
      "perplexity_per_byte_ablated": 12.707451238979461
    },
    {
      "seed": 4049,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.748289140065511,
      "eval_nll_ablated": 2.748289140065511,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.615892412295414,
      "perplexity_per_byte_ablated": 15.615892412295414
    },
    {
      "seed": 4049,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.616976245244344,
      "eval_nll_ablated": 2.616976245244344,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.694252855820134,
      "perplexity_per_byte_ablated": 13.694252855820134
    },
    {
      "seed": 4049,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5334832032521564,
      "eval_nll_ablated": 2.5334832032521564,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.597308792559115,
      "perplexity_per_byte_ablated": 12.597308792559115
    },
    {
      "seed": 4049,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.748961162567139,
      "eval_nll_ablated": 2.751382756233215,
      "label_use_gap": 0.002421593666076305,
      "perplexity_per_byte_labeled": 15.626390170349213,
      "perplexity_per_byte_ablated": 15.664276792296944
    },
    {
      "seed": 4049,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6176442464192706,
      "eval_nll_ablated": 2.6265236377716064,
      "label_use_gap": 0.00887939135233573,
      "perplexity_per_byte_labeled": 13.703403688860895,
      "perplexity_per_byte_ablated": 13.825623388325646
    },
    {
      "seed": 4049,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.523064613342285,
      "eval_nll_ablated": 2.5562360445658365,
      "label_use_gap": 0.03317143122355137,
      "perplexity_per_byte_labeled": 12.466743930193617,
      "perplexity_per_byte_ablated": 12.887218993996944
    },
    {
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.748648834228516,
      "eval_nll_ablated": 2.7515129725138348,
      "label_use_gap": 0.002864138285318951,
      "perplexity_per_byte_labeled": 15.621510367958587,
      "perplexity_per_byte_ablated": 15.666316668969126
    },
    {
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6160919109980267,
      "eval_nll_ablated": 2.627035554250081,
      "label_use_gap": 0.010943643252054436,
      "perplexity_per_byte_labeled": 13.682147912238968,
      "perplexity_per_byte_ablated": 13.83270276463341
    },
    {
      "seed": 4049,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5227548996607463,
      "eval_nll_ablated": 2.5585955301920573,
      "label_use_gap": 0.035840630531311035,
      "perplexity_per_byte_labeled": 12.462883406893496,
      "perplexity_per_byte_ablated": 12.917662102890665
    },
    {
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7487150192260743,
      "eval_nll_ablated": 2.7512818415959677,
      "label_use_gap": 0.0025668223698933623,
      "perplexity_per_byte_labeled": 15.622544311799558,
      "perplexity_per_byte_ablated": 15.662696117244671
    },
    {
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.61727348168691,
      "eval_nll_ablated": 2.6285632689793905,
      "label_use_gap": 0.011289787292480646,
      "perplexity_per_byte_labeled": 13.69832389182275,
      "perplexity_per_byte_ablated": 13.853851338773966
    },
    {
      "seed": 4049,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5187066793441772,
      "eval_nll_ablated": 2.5577756722768146,
      "label_use_gap": 0.039068992932637325,
      "perplexity_per_byte_labeled": 12.412532892832033,
      "perplexity_per_byte_ablated": 12.907075795596073
    },
    {
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7481045564015707,
      "eval_nll_ablated": 2.7503596623738606,
      "label_use_gap": 0.0022551059722899502,
      "perplexity_per_byte_labeled": 15.61301023966743,
      "perplexity_per_byte_ablated": 15.648258962168521
    },
    {
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.618207899729411,
      "eval_nll_ablated": 2.6266212622324625,
      "label_use_gap": 0.00841336250305158,
      "perplexity_per_byte_labeled": 13.711129834939664,
      "perplexity_per_byte_ablated": 13.826973173239871
    },
    {
      "seed": 4049,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.518013604482015,
      "eval_nll_ablated": 2.5431578874588014,
      "label_use_gap": 0.025144282976786503,
      "perplexity_per_byte_labeled": 12.403933058816882,
      "perplexity_per_byte_ablated": 12.719775233821485
    },
    {
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.748308459917704,
      "eval_nll_ablated": 2.750240707397461,
      "label_use_gap": 0.0019322474797567324,
      "perplexity_per_byte_labeled": 15.61619411194307,
      "perplexity_per_byte_ablated": 15.64639763460226
    },
    {
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.618260645866394,
      "eval_nll_ablated": 2.6264153718948364,
      "label_use_gap": 0.008154726028442472,
      "perplexity_per_byte_labeled": 13.711853063145709,
      "perplexity_per_byte_ablated": 13.824126626113216
    },
    {
      "seed": 4049,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.520236841837565,
      "eval_nll_ablated": 2.5465072631835937,
      "label_use_gap": 0.026270421346028527,
      "perplexity_per_byte_labeled": 12.431540623862421,
      "perplexity_per_byte_ablated": 12.762449967177556
    },
    {
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.74776775042216,
      "eval_nll_ablated": 2.7497911930084227,
      "label_use_gap": 0.002023442586262636,
      "perplexity_per_byte_labeled": 15.607752569919118,
      "perplexity_per_byte_ablated": 15.639365934272538
    },
    {
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6180982907613117,
      "eval_nll_ablated": 2.625451183319092,
      "label_use_gap": 0.007352892557780155,
      "perplexity_per_byte_labeled": 13.709627054507592,
      "perplexity_per_byte_ablated": 13.810803984952003
    },
    {
      "seed": 4049,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.521579869588216,
      "eval_nll_ablated": 2.5496434132258097,
      "label_use_gap": 0.028063543637593558,
      "perplexity_per_byte_labeled": 12.448247744455657,
      "perplexity_per_byte_ablated": 12.802537752979731
    },
    {
      "seed": 5051,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.726653774579366,
      "eval_nll_ablated": 2.726653774579366,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.281665470809237,
      "perplexity_per_byte_ablated": 15.281665470809237
    },
    {
      "seed": 5051,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.616872239112854,
      "eval_nll_ablated": 2.616872239112854,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.692828643621635,
      "perplexity_per_byte_ablated": 13.692828643621635
    },
    {
      "seed": 5051,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5487047910690306,
      "eval_nll_ablated": 2.5487047910690306,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.790526645209551,
      "perplexity_per_byte_ablated": 12.790526645209551
    },
    {
      "seed": 5051,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.725585865974426,
      "eval_nll_ablated": 2.7413246870040893,
      "label_use_gap": 0.015738821029663175,
      "perplexity_per_byte_labeled": 15.265354759479735,
      "perplexity_per_byte_ablated": 15.507514098478733
    },
    {
      "seed": 5051,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6193121671676636,
      "eval_nll_ablated": 2.6485416889190674,
      "label_use_gap": 0.02922952175140381,
      "perplexity_per_byte_labeled": 13.726278951956951,
      "perplexity_per_byte_ablated": 14.133412697159105
    },
    {
      "seed": 5051,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.547375186284383,
      "eval_nll_ablated": 2.5973735491434735,
      "label_use_gap": 0.04999836285909032,
      "perplexity_per_byte_labeled": 12.773531600633936,
      "perplexity_per_byte_ablated": 13.428422586154262
    },
    {
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.724604773521423,
      "eval_nll_ablated": 2.7421523650487263,
      "label_use_gap": 0.01754759152730312,
      "perplexity_per_byte_labeled": 15.250385379506378,
      "perplexity_per_byte_ablated": 15.520354640609309
    },
    {
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6202327728271486,
      "eval_nll_ablated": 2.65557226339976,
      "label_use_gap": 0.03533949057261143,
      "perplexity_per_byte_labeled": 13.738921260441284,
      "perplexity_per_byte_ablated": 14.23312882789636
    },
    {
      "seed": 5051,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.550892154375712,
      "eval_nll_ablated": 2.6191474119822185,
      "label_use_gap": 0.06825525760650653,
      "perplexity_per_byte_labeled": 12.818534794698435,
      "perplexity_per_byte_ablated": 13.724017662607437
    },
    {
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7251710891723633,
      "eval_nll_ablated": 2.7420976082483928,
      "label_use_gap": 0.01692651907602949,
      "perplexity_per_byte_labeled": 15.259024357392967,
      "perplexity_per_byte_ablated": 15.519504818916118
    },
    {
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.622125768661499,
      "eval_nll_ablated": 2.6549779812494916,
      "label_use_gap": 0.032852212587992646,
      "perplexity_per_byte_labeled": 13.764953612949428,
      "perplexity_per_byte_ablated": 14.224672846359784
    },
    {
      "seed": 5051,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.551601831118266,
      "eval_nll_ablated": 2.63835125764211,
      "label_use_gap": 0.08674942652384399,
      "perplexity_per_byte_labeled": 12.827635039449941,
      "perplexity_per_byte_ablated": 13.990118481318142
    },
    {
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.726101358731588,
      "eval_nll_ablated": 2.740979226430257,
      "label_use_gap": 0.014877867698669167,
      "perplexity_per_byte_labeled": 15.273225967894913,
      "perplexity_per_byte_ablated": 15.502157789009702
    },
    {
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6213167746861776,
      "eval_nll_ablated": 2.644200658798218,
      "label_use_gap": 0.02288388411204023,
      "perplexity_per_byte_labeled": 13.753822351574762,
      "perplexity_per_byte_ablated": 14.072192103290817
    },
    {
      "seed": 5051,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.550094970067342,
      "eval_nll_ablated": 2.6118900934855143,
      "label_use_gap": 0.06179512341817217,
      "perplexity_per_byte_labeled": 12.8083201319292,
      "perplexity_per_byte_ablated": 13.62477863555408
    },
    {
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.725763463973999,
      "eval_nll_ablated": 2.7401273250579834,
      "label_use_gap": 0.014363861083984464,
      "perplexity_per_byte_labeled": 15.268066096704695,
      "perplexity_per_byte_ablated": 15.488957103155592
    },
    {
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6223496437072753,
      "eval_nll_ablated": 2.643641487757365,
      "label_use_gap": 0.021291844050089548,
      "perplexity_per_byte_labeled": 13.768035587545366,
      "perplexity_per_byte_ablated": 14.06432554056734
    },
    {
      "seed": 5051,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5536568721135455,
      "eval_nll_ablated": 2.6090280373891193,
      "label_use_gap": 0.05537116527557373,
      "perplexity_per_byte_labeled": 12.854023460685264,
      "perplexity_per_byte_ablated": 13.585839504367444
    },
    {
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7252547899882,
      "eval_nll_ablated": 2.7434869527816774,
      "label_use_gap": 0.018232162793477613,
      "perplexity_per_byte_labeled": 15.260301603633081,
      "perplexity_per_byte_ablated": 15.541081743515894
    },
    {
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6216513395309446,
      "eval_nll_ablated": 2.650778539975484,
      "label_use_gap": 0.029127200444539536,
      "perplexity_per_byte_labeled": 13.75842466685829,
      "perplexity_per_byte_ablated": 14.165062420943942
    },
    {
      "seed": 5051,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.552683409055074,
      "eval_nll_ablated": 2.6183469692866006,
      "label_use_gap": 0.06566356023152675,
      "perplexity_per_byte_labeled": 12.841516632148991,
      "perplexity_per_byte_ablated": 13.713036768289493
    },
    {
      "seed": 6067,
      "condition": "none",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.732598559061686,
      "eval_nll_ablated": 2.732598559061686,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 15.372782245014543,
      "perplexity_per_byte_ablated": 15.372782245014543
    },
    {
      "seed": 6067,
      "condition": "none",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6095396439234415,
      "eval_nll_ablated": 2.6095396439234415,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 13.59279188692266,
      "perplexity_per_byte_ablated": 13.59279188692266
    },
    {
      "seed": 6067,
      "condition": "none",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5251223405202228,
      "eval_nll_ablated": 2.5251223405202228,
      "label_use_gap": 0.0,
      "perplexity_per_byte_labeled": 12.4924234997121,
      "perplexity_per_byte_ablated": 12.4924234997121
    },
    {
      "seed": 6067,
      "condition": "hierarchy",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.735201056798299,
      "eval_nll_ablated": 2.7932584126790365,
      "label_use_gap": 0.05805735588073757,
      "perplexity_per_byte_labeled": 15.412841981088503,
      "perplexity_per_byte_ablated": 16.33415660694414
    },
    {
      "seed": 6067,
      "condition": "hierarchy",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6137526114781697,
      "eval_nll_ablated": 2.7099109808603923,
      "label_use_gap": 0.09615836938222255,
      "perplexity_per_byte_labeled": 13.650178677583723,
      "perplexity_per_byte_ablated": 15.027937681247423
    },
    {
      "seed": 6067,
      "condition": "hierarchy",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.528871949513753,
      "eval_nll_ablated": 2.641370757420858,
      "label_use_gap": 0.11249880790710476,
      "perplexity_per_byte_labeled": 12.539353132119295,
      "perplexity_per_byte_ablated": 14.032425481822393
    },
    {
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7355334122975665,
      "eval_nll_ablated": 2.796801471710205,
      "label_use_gap": 0.061268059412638465,
      "perplexity_per_byte_labeled": 15.417965375227208,
      "perplexity_per_byte_ablated": 16.3921321327335
    },
    {
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6143938144048056,
      "eval_nll_ablated": 2.716135867436727,
      "label_use_gap": 0.1017420530319213,
      "perplexity_per_byte_labeled": 13.658934018776115,
      "perplexity_per_byte_ablated": 15.121776654255788
    },
    {
      "seed": 6067,
      "condition": "scrambled_1",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5292214790980023,
      "eval_nll_ablated": 2.6610300461451213,
      "label_use_gap": 0.13180856704711896,
      "perplexity_per_byte_labeled": 12.54373677306779,
      "perplexity_per_byte_ablated": 14.311022522903974
    },
    {
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7363977273305258,
      "eval_nll_ablated": 2.7973005612691244,
      "label_use_gap": 0.06090283393859863,
      "perplexity_per_byte_labeled": 15.431297115060293,
      "perplexity_per_byte_ablated": 16.400315316630333
    },
    {
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6138134797414145,
      "eval_nll_ablated": 2.7204511006673178,
      "label_use_gap": 0.10663762092590323,
      "perplexity_per_byte_labeled": 13.651009565539905,
      "perplexity_per_byte_ablated": 15.187171643216606
    },
    {
      "seed": 6067,
      "condition": "scrambled_2",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5287177165349326,
      "eval_nll_ablated": 2.646742510795593,
      "label_use_gap": 0.11802479426066048,
      "perplexity_per_byte_labeled": 12.537419299467466,
      "perplexity_per_byte_ablated": 14.108007031736332
    },
    {
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.736346896489461,
      "eval_nll_ablated": 2.794234053293864,
      "label_use_gap": 0.05788715680440282,
      "perplexity_per_byte_labeled": 15.430512749184372,
      "perplexity_per_byte_ablated": 16.35010065010213
    },
    {
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6144343614578247,
      "eval_nll_ablated": 2.7110803604125975,
      "label_use_gap": 0.09664599895477277,
      "perplexity_per_byte_labeled": 13.65948785952619,
      "perplexity_per_byte_ablated": 15.04552132325567
    },
    {
      "seed": 6067,
      "condition": "scrambled_3",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.531559181213379,
      "eval_nll_ablated": 2.649282375971476,
      "label_use_gap": 0.117723194758097,
      "perplexity_per_byte_labeled": 12.573094594608003,
      "perplexity_per_byte_ablated": 14.143885010825038
    },
    {
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.7351391553878783,
      "eval_nll_ablated": 2.792897335688273,
      "label_use_gap": 0.05775818030039481,
      "perplexity_per_byte_labeled": 15.41188793396002,
      "perplexity_per_byte_ablated": 16.328259783497547
    },
    {
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.6144195874532064,
      "eval_nll_ablated": 2.7098724921544393,
      "label_use_gap": 0.09545290470123291,
      "perplexity_per_byte_labeled": 13.659286055680198,
      "perplexity_per_byte_ablated": 15.027359286503833
    },
    {
      "seed": 6067,
      "condition": "scrambled_4",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.5311453024546307,
      "eval_nll_ablated": 2.6451526085535684,
      "label_use_gap": 0.11400730609893772,
      "perplexity_per_byte_labeled": 12.567891934533073,
      "perplexity_per_byte_ablated": 14.085594501313436
    },
    {
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 200,
      "examples_processed": 4800,
      "eval_nll_labeled": 2.735658566157023,
      "eval_nll_ablated": 2.7955135345458983,
      "label_use_gap": 0.059854968388875385,
      "perplexity_per_byte_labeled": 15.419895113853483,
      "perplexity_per_byte_ablated": 16.371033686210698
    },
    {
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 400,
      "examples_processed": 9600,
      "eval_nll_labeled": 2.613860352834066,
      "eval_nll_ablated": 2.712605690956116,
      "label_use_gap": 0.09874533812205,
      "perplexity_per_byte_labeled": 13.651649445572515,
      "perplexity_per_byte_ablated": 15.068488228080934
    },
    {
      "seed": 6067,
      "condition": "scrambled_5",
      "step": 800,
      "examples_processed": 19200,
      "eval_nll_labeled": 2.535284646352132,
      "eval_nll_ablated": 2.66018758614858,
      "label_use_gap": 0.12490293979644784,
      "perplexity_per_byte_labeled": 12.620022580217201,
      "perplexity_per_byte_ablated": 14.298971136037272
    }
  ]
}