fix train_lm entry

stanford-crfm · Sep 16, 2024 · 3f60b82 · 3f60b82
1 parent 0827737
commit 3f60b82
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/tests/tiny_test_corpus.py b/tests/tiny_test_corpus.py
@@ -69,7 +69,7 @@ def construct_small_data_cache(
         validation_urls=[f"file://{path}/validation/docs.jsonl"],
         cache_dir=f"{path}/cache",
         vocab_size=vocab_size,
-        tokenizer="passthrough",
+        tokenizer="gpt2",
     )
 
     return config, caches