efficientnet not complete, will come back

examples/alexnet.py

@@ -32,13 +32,13 @@ def __init__(self, num_classes: int = 1000, dropout: float = 0.5) -> None:
         )
         self.avgpool = nn.AdaptiveAvgPool2d(6)
         self.classifier = nn.Sequential(
-            # nn.Dropout(dropout),
+            # nn.Dropout(p=dropout),
             nn.Linear(256 * 6 * 6, 4096),
             nn.ReLU(),
-            # nn.Dropout(dropout),
+            # nn.Dropout(p=dropout),
             nn.Linear(4096, 4096),
             nn.ReLU(),
-            nn.Linear(4096, 1000),
+            nn.Linear(4096, num_classes),
         )
 
     def forward(self, x: Tensor) -> Tensor:
@@ -48,7 +48,7 @@ def forward(self, x: Tensor) -> Tensor:
         x = self.classifier(x)
         return x
 
-    def load_weights(self, weights: Sequence[Tensor]) -> None:
+    def from_pretrained(self, weights: Sequence[Tensor]) -> None:
         index = 0
         trainable_layers = [
             l for l in self.features.layers + self.classifier.layers if l.name in ["Conv2d", "Linear"]
@@ -69,7 +69,7 @@ def load_weights_from_pytorch(save_directory="./weights"):
         with open(save_path, "wb") as file:
             file.write(response.content)
     # parse it into a list of Tensors using torch
-    weights = []
+    weights: list = []
     state_dict = torch.load(save_path)
     for key, value in state_dict.items():
         # print(f"Key: {key}, Tensor Shape: {value.shape}")
@@ -111,7 +111,7 @@ def pytorch_alexnet(img):
 def gradipy_alexnet(img):
     alexnet = AlexNet()
     weights = load_weights_from_pytorch()
-    alexnet.load_weights(weights)
+    alexnet.from_pretrained(weights)
     logits = alexnet(Tensor(img.numpy()))
     idx = np.argmax(logits.data, axis=1)[0]
     value = np.max(logits.data, axis=1)[0]
@@ -130,4 +130,3 @@ def gradipy_alexnet(img):
     print(f"GradiPy: {idx_=}, {value_=}, {cls_=}")
     np.testing.assert_allclose(logits.data, logits_.data, atol=1e-4)
     assert idx == idx_ and int(value) == int(value_) and cls == cls_
-    load_weights_from_pytorch()

examples/efficientnet.py

@@ -0,0 +1,29 @@
+import torch
+import numpy as np
+from gradipy.tensor import Tensor
+import gradipy.nn as nn
+import gradipy.nn.functional as F
+
+
+def load_weights_from_pytorch(path="./weights/efficientnet-b0-355c32eb.pth"):
+    state_dict = torch.load(path)
+    for key, value in state_dict.items():
+        print(f"Key: {key}, --> Tensor Shape: {value.shape}")
+
+
+batch_norm_momentum = 0.9
+batch_norm_epsilon = 1e-5
+has_se = None
+id_skip = None
+inp = 3
+oup = 1000
+image_size = 224
+
+
+class MBConvBlock(nn.Module):
+    def __init__(self) -> None:
+        super().__init__()
+
+
+if __name__ == "__main__":
+    load_weights_from_pytorch()

examples/efficientnet_args.py

@@ -0,0 +1,86 @@
+blocks_args = [
+    {
+        "num_repeat": 1,
+        "kernel_size": 3,
+        "stride": [1],
+        "expand_ratio": 1,
+        "input_filters": 32,
+        "output_filters": 16,
+        "se_ratio": 0.25,
+        "id_skip": True,
+    },
+    {
+        "num_repeat": 2,
+        "kernel_size": 3,
+        "stride": [2],
+        "expand_ratio": 6,
+        "input_filters": 16,
+        "output_filters": 24,
+        "se_ratio": 0.25,
+        "id_skip": True,
+    },
+    {
+        "num_repeat": 2,
+        "kernel_size": 5,
+        "stride": [2],
+        "expand_ratio": 6,
+        "input_filters": 24,
+        "output_filters": 40,
+        "se_ratio": 0.25,
+        "id_skip": True,
+    },
+    {
+        "num_repeat": 3,
+        "kernel_size": 3,
+        "stride": [2],
+        "expand_ratio": 6,
+        "input_filters": 40,
+        "output_filters": 80,
+        "se_ratio": 0.25,
+        "id_skip": True,
+    },
+    {
+        "num_repeat": 3,
+        "kernel_size": 5,
+        "stride": [1],
+        "expand_ratio": 6,
+        "input_filters": 80,
+        "output_filters": 112,
+        "se_ratio": 0.25,
+        "id_skip": True,
+    },
+    {
+        "num_repeat": 4,
+        "kernel_size": 5,
+        "stride": [2],
+        "expand_ratio": 6,
+        "input_filters": 112,
+        "output_filters": 192,
+        "se_ratio": 0.25,
+        "id_skip": True,
+    },
+    {
+        "num_repeat": 1,
+        "kernel_size": 3,
+        "stride": [1],
+        "expand_ratio": 6,
+        "input_filters": 192,
+        "output_filters": 320,
+        "se_ratio": 0.25,
+        "id_skip": True,
+    },
+]
+
+global_params = {
+    "width_coefficient": 1.0,
+    "depth_coefficient": 1.0,
+    "image_size": 224,
+    "dropout_rate": 0.2,
+    "num_classes": 1000,
+    "batch_norm_momentum": 0.99,
+    "batch_norm_epsilon": 0.001,
+    "drop_connect_rate": 0.2,
+    "depth_divisor": 8,
+    "min_depth": None,
+    "include_top": True,
+}

gradipy/tensor.py

@@ -83,6 +83,15 @@ def _backward() -> None:
         out._backward = _backward
         return out
 
+    def silu(self) -> "Tensor":
+        out = Tensor(self.data * (1.0 / (1.0 + np.exp(-self.data))))
+
+        def _backward() -> None:
+            pass
+
+        out._backward = _backward
+        return out
+
     def dropout(self: "Tensor", p: float = 0.5):
         if p < 0 or p > 1:
             raise ValueError(f"dropout probability should be between 0 and 1, but got {p}")
@@ -249,7 +258,7 @@ def _backward() -> None:
         return out
 
     def backward(self) -> None:
-        """Autograd engince that does topological sort to call `.backward()`"""
+        """Autograd engine that does topological sort to call `.backward()`"""
         topo = []
         visited = set()