idk where the leak is

Author: nico-byte

Network.mojo

@@ -60,7 +60,7 @@ struct Network:
 
     @staticmethod
     fn relu(A: Matrix) -> Matrix:
-        var B: Matrix = Matrix(A.rows, A.cols, True)
+        var B: Matrix = Matrix(A.rows, A.cols)
         for i in range(B.rows):
             for j in range(B.cols):
                 if A[i, j] > 0.01:
@@ -71,7 +71,7 @@ struct Network:
 
     @staticmethod
     fn drelu(A: Matrix) -> Matrix:
-        var B: Matrix = Matrix(A.rows, A.cols, True)
+        var B: Matrix = Matrix(A.rows, A.cols)
         for i in range(B.rows):
             for j in range(B.cols):
                 if A[i, j] > 0.01:
@@ -83,7 +83,7 @@ struct Network:
     @staticmethod
     fn tanh(A: Matrix) -> Matrix:
         # could need optimization alot
-        var B: Matrix = Matrix(A.rows, A.cols, True)
+        var B: Matrix = Matrix(A.rows, A.cols)
 
         for i in range(A.rows):
             for j in range(A.cols):
@@ -93,7 +93,7 @@ struct Network:
     @staticmethod
     fn dtanh(A: Matrix) -> Matrix:
         # could need optimization alot
-        var B: Matrix = Matrix(A.rows, A.cols, True)
+        var B: Matrix = Matrix(A.rows, A.cols)
 
         for i in range(A.rows):
             for j in range(A.cols):
@@ -103,9 +103,9 @@ struct Network:
     @staticmethod
     fn softmax_1d(A: Matrix) -> Matrix:
         # could need optimization alot
-        var B: Matrix = Matrix(A.rows, A.cols, True)
+        var B: Matrix = Matrix(A.rows, A.cols)
 
-        var row_exp_sum_mat: Matrix = Matrix(A.rows, 1, True)
+        var row_exp_sum_mat: Matrix = Matrix(A.rows, 1)
         for i in range(A.rows):
             for j in range(A.cols):
                     B[i, j] += exp(A[i, j])
@@ -141,19 +141,16 @@ struct Network:
 
         let time_now = now()
         # calc output hidden layer1
-        inputs_h1.zero()
         nj.matmul_vectorized(inputs_h1, inputs, self._wih)
         inputs_h1 = inputs_h1 + self._bih_l1
         inputs_h1 = self.relu(inputs_h1)
 
         # calc output hidden layer 2
-        inputs_h2.zero()
         nj.matmul_vectorized(inputs_h2, inputs_h1, self._whh)
         inputs_h2 = inputs_h2 + self._bih_l2
         inputs_h2 = self.tanh(inputs_h2)
 
         # calc output output layer
-        outputs.zero()
         nj.matmul_vectorized(outputs, inputs_h2, self._who)
         outputs = outputs + self._bho
         outputs = self.softmax_1d(outputs)
@@ -194,11 +191,8 @@ struct Network:
         var ih1_ho2: Matrix = Matrix(inputs_h1.cols, ho2_drelu.cols)
         var i_ho1: Matrix = Matrix(inputs.cols, ho1_drelu.cols)
 
-        ih2_o.zero()
         nj.matmul_vectorized(ih2_o, inputs_h2.transpose(), output_error_gradient)
-        ih1_ho2.zero()
         nj.matmul_vectorized(ih1_ho2, inputs_h1.transpose(), ho2_drelu)
-        i_ho1.zero()
         nj.matmul_vectorized(i_ho1, inputs.transpose(), ho1_drelu)
 
         # updating weights and biases

main.mojo

@@ -4,21 +4,21 @@ from python import Python
 from time import now
 import numjo as nj
 from random import randn, rand
+from matmul import matmul_benchmark
+
 
 fn main() raises:
     let input_nodes = 784
     let hidden_nodes_1 = 150
     let hidden_nodes_2 = 80
     let output_nodes = 10
     let learning_rate = 1e-4
-    var outputs = Matrix(1, output_nodes, True)
-    
-    var peval_nn = Network(input_nodes=input_nodes, hidden_nodes_l1=hidden_nodes_1, hidden_nodes_l2=hidden_nodes_2, output_nodes=output_nodes, learning_rate=learning_rate)
-    # var train_nn = Network(input_nodes=input_nodes, hidden_nodes_l1=hidden_nodes_1, hidden_nodes_l2=hidden_nodes_2, output_nodes=output_nodes, learning_rate=learning_rate)
+    var outputs = Matrix(1, output_nodes)
 
     # download dataset first - https://www.kaggle.com/datasets/oddrationale/mnist-in-csv
     Python.add_to_path("./")
     let DataLoader = Python.import_module("DataLoader")
+    let py_matmul = Python.import_module("py_matmul")
     let np = Python.import_module("numpy")
 
     print("\nStarting python mnist data loader")
@@ -34,11 +34,11 @@ fn main() raises:
     mnist_train_labels  = DataLoader.mnist_labels("train", output_nodes)
     mnist_test_labels = DataLoader.mnist_labels("", output_nodes)
 
-    let test_inputs: Matrix = Matrix(10000, 784, True)
-    let test_labels: Matrix = Matrix(10000, 10, True)
+    let test_inputs: Matrix = Matrix(10000, 784)
+    let test_labels: Matrix = Matrix(10000, 10)
 
-    let train_inputs: Matrix = Matrix(60000, 784, True)
-    let train_labels: Matrix = Matrix(60000, 10, True)
+    let train_inputs: Matrix = Matrix(60000, 784)
+    let train_labels: Matrix = Matrix(60000, 10)
 
     print("Starting train data converter")
     for i in range(mnist_test_inputs.shape[0]):
@@ -62,16 +62,21 @@ fn main() raises:
         for j in range(mnist_test_labels.shape[1]):
             test_labels[i, j] = mnist_test_labels[i][j].to_float64().cast[DType.float32]()
 
+    py_matmul.py_matmul_benchmark()
+    matmul_benchmark()
 
-    var mse: Matrix = Matrix(1, 1, True)
+    var peval_nn = Network(input_nodes=input_nodes, hidden_nodes_l1=hidden_nodes_1, hidden_nodes_l2=hidden_nodes_2, output_nodes=output_nodes, learning_rate=learning_rate)
+    # var train_nn = Network(input_nodes=input_nodes, hidden_nodes_l1=hidden_nodes_1, hidden_nodes_l2=hidden_nodes_2, output_nodes=output_nodes, learning_rate=learning_rate)
+    
+    var mse: Matrix = Matrix(1, 1)
 
     print("Start evaluating performance")
-    var iter_time: Matrix = Matrix(1, 1, True)
+    var iter_time: Matrix = Matrix(1, 1)
     var time_sum: Float32 = 0.0
     var iter: Int = 0
 
-    var new_input: Matrix = Matrix(1, input_nodes, True)
-    var new_label: Matrix = Matrix(1, output_nodes, True)
+    var new_input: Matrix = Matrix(1, input_nodes)
+    var new_label: Matrix = Matrix(1, output_nodes)
 
     var time_now = now()
     for i in range(test_inputs.rows):

matmul.mojo

@@ -2,7 +2,7 @@ from numjo import matmul_vectorized, matmul, Matrix
 from random import rand
 from time import now
 
-fn main() -> None:
+fn matmul_benchmark() -> None:
     let M: Int = 512
     let K: Int = 512
 

matmul.py

@@ -1,4 +1,4 @@
-from memory import Pointer
+from memory.unsafe import Pointer
 from memory import memset_zero
 from random import randn, rand, seed
 from algorithm import vectorize
@@ -19,10 +19,9 @@ struct Matrix:
     var cols: Int
 
     # Initialize
-    fn __init__(inout self, rows: Int, cols: Int, zero:Bool = False):
+    fn __init__(inout self, rows: Int, cols: Int):
         self.data = DTypePointer[type].alloc(rows * cols)
-        if zero:    
-            memset_zero(self.data, rows * cols)
+        memset_zero(self.data, rows * cols)
         self.rows = rows
         self.cols = cols
 

numjo/matrix/matrix.mojo

@@ -0,0 +1,12 @@
+import numpy as np
+from time import time
+
+def py_matmul_benchmark():
+    M = 512
+    K = 512
+    A = np.random.randn(M , K)
+    B = np.random.randn(K, M)
+    start_time = time()
+    C = np.dot(A, B)
+    end_time = time()-start_time
+    print(f"Matrix Multiplication in Python with 512x512 matrices took {end_time * 1e3} milliseconds.")

py_matmul.py

@@ -66,24 +66,20 @@ fn main() -> None:
     nj.matmul(mul_matrix_C, mul_matrix_A, mul_matrix_B)
     mul_matrix_C.print_all()
 
-    let A: Matrix = Matrix(1, 50)
-    rand(A.data, A.rows * A.cols)
+    let A: Matrix = Matrix(2, 50)
+    randn(A.data, A.rows * A.cols)
     A.print_all()
 
-    let B: Matrix = Matrix(50, 1)
-    rand(B.data, B.rows * B.cols)
+    let B: Matrix = Matrix(50, 2)
+    randn(B.data, B.rows * B.cols)
     A.print_all()
 
     let C: Matrix = Matrix(A.rows, B.cols)
-    nj.matmul(C, A, B)
+    nj.matmul_vectorized(C, A, B)
     C.print_all()
 
     A[0, 35] = 0.99
     print(nj.argmax(A))
 
     print(nj.mean(A))
     print(nj.sum(A))
-
-    
-
-