data-science-from-scratch/DeepNeuralNetwork.py at master · lkarjun/data-science-from-scratch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
from typing import Dict, List
import numpy as np
from numpy.core.numeric import full


class DeepNeuralNetwork:

    def __init__(self, X: np.array, Y: np.array, hidden_layers: List[int], activations: List[str]) -> None:
        assert len(activations) != len(hidden_layers), "\n>>> hidden_layer = [10,20];\n>>> activations = [relu, tanh, sigmoid];\nMention final layer activation"
        self.X = X
        self.Y = Y
        self.hidden_layers: List[int] = hidden_layers
        self.activation: List[str] = activations
        self.param = None
        self.Layer = None
        self.cache = None
        self.grads = None

    def initialize_parameters(self) -> Dict[str, np.array]:
        Layers = self.layer_size()
        parameters: Dict[str, np.array] = {}

        for i in range(1, len(Layers)):
            parameters["W"+str(i)] = np.random.randn(Layers[i], Layers[i-1]) * 0.1
            parameters["b"+str(i)] = np.zeros(shape=(Layers[i], 1))

        self.param = parameters
        return parameters

    def layer_size(self) -> List[int]:
        Layer_0 = [self.X.shape[0]]
        Layer_last = [self.Y.shape[0]]
        Layer_hidden = self.hidden_layers
        full_layer = (Layer_0 + Layer_hidden) + Layer_last
        self.Layer = full_layer
        return full_layer

    def forward_pass(self) -> Dict[str, np.array]:
        cache: Dict[str, np.array] = {}
        cache['Z1'] = (np.dot(self.param['W1'],
                        self.X)) + self.param['b1']
        cache['A1'] = self.Activation(data = cache['Z1'], activation = self.activation[0])
        cache['A1_activation'] = self.activation[0]

        for i in range(2, len(self.Layer)):
            cache['Z'+str(i)] = (np.dot(self.param['W'+str(i)],
                                    cache['A'+str(i-1)])) + self.param['b'+str(i)]
            # cache['A'+str(i)] = np.tanh(cache['Z'+str(i)])
            cache['A'+str(i)] = self.Activation(data = cache['Z'+str(i)], activation = self.activation[i-1])
            cache['A'+str(i)+'_activation'] = self.activation[i-1]

        self.cache = cache
        return cache


    def backward_pass(self):
        m = self.X.shape[1]
        grads: Dict[str, np.array] = {}
        i = len(self.layer_size()) - 1

        while i > 1:
            grads["dz"+str(i)] = self.cache['A'+str(i)] - self.Y
            grads["dw"+str(i)] = 1/m * np.dot(grads['dz'+str(i)], self.cache['A'+str(i-1)].T)
            grads["db"+str(i)] = 1/m * np.sum(grads['dz'+str(i)], axis=1, keepdims=True)
            i -= 1

        grads['dz1'] = self.cache['A1'] - self.Y
        grads['dw1'] = 1/m * np.dot(grads['dz1'], self.X.T)
        grads['db1'] = 1/m * np.sum(grads['dz1'], axis=1, keepdims=True)

        self.grads = grads

        return grads

    def update_parameters(self):
        ...

    def fit(self):
        ...

    def predict(self):
        ...

    def Activation(self, data: np.array, activation: str) -> np.array:

        def sigmoid(z: np.array) -> np.array:
            return 1/(1+np.exp(-z))

        def tanh(z: np.array) -> np.array:
            return np.tanh(z)

        if activation == 'sigmoid': return sigmoid(data)
        if activation == 'tanh': return tanh(data)


    def derivative_of_Activation(self, data: np.array, Activation: str) -> np.array:
        ...

sample_data = np.array([[1,2,3],[2, 6, 8]]).reshape(2,-1)
sample_y = np.array([[0], [1], [3]]).reshape(1, -1)
model = DeepNeuralNetwork(sample_data, sample_y, [10, 10], ['tanh', 'tanh', 'sigmoid'])

print(model.layer_size())
from pprint import pprint
pprint(model.initialize_parameters())

pprint(model.forward_pass())
print(len(model.layer_size()) - 1)
print(model.layer_size())
print(sample_y)

pprint(model.backward_pass())