feat: support silu

Author: RayZhao1998

cs336_basics/silu.py

@@ -0,0 +1,6 @@
+from jaxtyping import Float
+from torch import Tensor, exp
+
+
+def SiLU(in_features: Float[Tensor, "..."]) -> Float[Tensor, "..."]:
+    return in_features * (1 / (1 + exp(-in_features)))

tests/adapters.py

@@ -27,6 +27,7 @@
 from cs336_basics.gradient_clipping import gradient_clipping
 from cs336_basics.data_loading import data_loading
 from cs336_basics.checkpointing import save_checkpoint, load_checkpoint
+from cs336_basics.silu import SiLU
 
 def run_linear(
     d_in: int,
@@ -462,7 +463,7 @@ def run_silu(in_features: Float[Tensor, " ..."]) -> Float[Tensor, " ..."]:
         Float[Tensor,"..."]: of with the same shape as `in_features` with the output of applying
         SiLU to each element.
     """
-    raise NotImplementedError
+    return SiLU(in_features)
 
 
 def run_get_batch(

tests/test_train_bpe.py

@@ -123,7 +123,7 @@ def _save_tokenizer_artifacts(vocab, merges, output_dir):
             right_unicode = ''.join([byte_to_unicode[byte] for byte in right])
             merges_file.write(f"['{left_unicode}', '{right_unicode}']\n")
 
-# @pytest.mark.skip()
+@pytest.mark.skip()
 def test_train_bpe_on_tiny_story_valid():
     start_time = time.time()
     input_path = data_folder / "TinyStoriesV2-GPT4-valid.txt"
@@ -136,7 +136,7 @@ def test_train_bpe_on_tiny_story_valid():
 
     assert(end_time - start_time <= 120)
 
-# @pytest.mark.skip()
+@pytest.mark.skip()
 def test_train_bpe_on_tiny_story_train():
     start_time = time.time()
     input_path = data_folder / "TinyStoriesV2-GPT4-train.txt"