init

.gitignore

@@ -138,3 +138,4 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
+sgd_hw/

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.pythonPath": "sgd_hw\\Scripts\\python.exe"
+}

layer.py

@@ -0,0 +1,192 @@
+import numpy as np
+from contextlib import contextmanager
+from typing import Generator, Dict, Union
+import io
+
+#only scalar gradient
+#op must be tree. 그래프 구현할려면, 위상정렬해서 순회해야하기 때문에 그렇게 하지 않음.
+
+class NonExistVarableError(ValueError):
+    pass
+
+def make_mermaid_graph(result):
+    with io.StringIO("") as graph:
+        graph.write("graph TD\n")
+        result.mermaid_graph(graph)
+        graph.write(f"{id(result)}-->Result\n")
+        return graph.getvalue()
+
+class OpTree:
+    def __init__(self):
+        super().__init__()
+    def __matmul__(self,a):
+        return MatMulOp(self,a)
+    def __add__(self,a):
+        return AddOp(self,a)
+    
+    @property
+    def T(self):
+        return TransposeOp(self)
+
+class MatMulOp(OpTree):
+    def __init__(self,a,b):
+        super().__init__()
+        self.a = a
+        self.b = b
+        va = self.a.numpy() if isinstance(self.a,OpTree) else self.a
+        vb = self.b.numpy() if isinstance(self.b,OpTree) else self.b
+        self.v = va @ vb
+    
+    def __str__(self):
+        return f"MatmulOp"
+
+    def mermaid_graph(self,writer):
+        if isinstance(self.a,OpTree):
+            self.a.mermaid_graph(writer)
+            writer.write(f'{id(self.a)}-->{id(self)}[MatmulOp]\n')
+        if isinstance(self.b,OpTree):
+            self.b.mermaid_graph(writer)
+            writer.write(f'{id(self.b)}-->{id(self)}[MatmulOp]\n')
+
+    def numpy(self):
+        return self.v
+
+    def backprop(self,seed):
+        #a @ b
+        a = self.a.numpy() if isinstance(self.a,OpTree) else self.a
+        b = self.b.numpy() if isinstance(self.b,OpTree) else self.b
+        if isinstance(self.a,OpTree):
+            s = seed * np.transpose(b) if seed.shape == () else  (seed) @ np.transpose(b)
+            #print('seed : ', s)
+            self.a.backprop((s))
+        if isinstance(self.b,OpTree):
+            s = np.transpose(a) * seed if seed.shape == () else np.transpose(a) @ seed
+            #print('seed : ', s) 
+            self.b.backprop(s)
+
+def matmul(a,b):
+    return MatMulOp(a,b)
+
+class AddOp(OpTree):
+    def __init__(self,a,b):
+        super().__init__()
+        self.a = a
+        self.b = b
+        va = self.a.numpy() if isinstance(self.a,OpTree) else self.a
+        vb = self.b.numpy() if isinstance(self.b,OpTree) else self.b
+        self.v = va + vb
+    
+    def __str__(self):
+        return f"AddOp"
+
+    def mermaid_graph(self,writer):
+        if isinstance(self.a,OpTree):
+            self.a.mermaid_graph(writer)
+            writer.write(f'{id(self.a)}-->{id(self)}[AddOp]\n')
+        if isinstance(self.b,OpTree):
+            self.b.mermaid_graph(writer)
+            writer.write(f'{id(self.b)}-->{id(self)}[AddOp]\n')
+
+    def numpy(self):
+        return self.v
+
+    def backprop(self,seed):
+        #borad_casted = self.a.shape != self.b.shape
+        #np.ones((1,b.shape[1]))
+        #a + b
+        if isinstance(self.a,OpTree):
+            self.a.backprop(seed)
+        if isinstance(self.b,OpTree):
+            self.b.backprop(seed)
+
+def addmul(a,b):
+    return AddOp(a,b)
+
+class FunctionOp(OpTree):
+    def __init__(self,f, f_grad, f_name, i):
+        super().__init__()
+        self.f = np.vectorize(f)
+        self.f_grad = np.vectorize(f_grad)
+        self.f_name = f_name
+        self.i = i
+        self.v = self.f(i.numpy())
+    
+    def __str__(self):
+        return f"Function{self.f_name}Op"
+
+    def mermaid_graph(self,writer):
+        self.i.mermaid_graph(writer)
+        writer.write(f'{id(self.i)}-->{id(self)}[Function{self.f_name}Op]\n')
+
+    def numpy(self):
+        return self.v
+
+    def backprop(self,seed):
+        self.i.backprop(seed * (self.f_grad(self.i.numpy())))
+
+class TransposeOp(OpTree):
+    def __init__(self, i):
+        super().__init__()
+        self.i = i
+        self.v = np.transpose(i.numpy())
+    
+    def __str__(self):
+        return f"TransposeOp"
+
+    def mermaid_graph(self,writer):
+        self.i.mermaid_graph(writer)
+        writer.write(f'{id(self.i)}-->{id(self)}[TransposeOp]\n')
+
+    def numpy(self):
+        return self.v
+
+    def backprop(self,seed):
+        self.i.backprop(np.transpose(seed))
+
+def transposemul(a):
+    return TransposeOp(a)
+
+def relu(v):
+    relu_f = lambda x: np.max([x,0])
+    relu_diff = lambda x: 1 if x > 0 else 0
+    return FunctionOp(relu_f,relu_diff,"Relu",v)
+    
+class Variable(OpTree):
+    def __init__(self,x):
+        super().__init__()
+        self.x = x
+        self.grad = None
+
+    def numpy(self):
+        return self.x
+    def mermaid_graph(self,writer):
+        writer.write(f'{id(self)}["Variable{self.x.shape}"]\n')
+    def backprop(self,seed):
+        self.grad = seed
+
+
+"""
+input_var = Variable(np.array([[1],[2],[3]]))
+weight = Variable(np.array([[2,-1,1]]))
+v = relu(weight @ input_var)
+print(f"result : {v.numpy()}")
+v.backprop(np.ones(()))
+print(f"grad input : {input_var.grad}, w : {weight.grad}")
+"""
+
+#input_diff = Variable(np.array([[1.01],[2],[3]]))
+#v_diff = relu(weight @ input_diff)
+#print(f"diff 1 : {(np.sum(v_diff.numpy()) - v.numpy()) / 0.01}")
+
+#i -= grad * delta
+
+"""
+graph TD
+2284612545696["Variable(1, 3)"]
+2284612545696-->2284612624880[MatmulOp]
+2284612544496["Variable(3, 2)"]
+2284612544496-->2284612624880[MatmulOp]
+2284612624880-->2284612625072[FunctionReluOp]
+2284612625072-->2284612627856[MatmulOp]
+2284612627856-->Result
+"""

p2.py

@@ -0,0 +1,32 @@
+from layer import *
+import numpy as np
+import pickle
+
+DIMENTION = 3
+VAR_RANGE = 1
+N = 10
+ALPHA = 0.1
+SIGMA = VAR_RANGE * ALPHA
+
+gen: np.random.Generator = np.random.default_rng()
+or_weight = (gen.uniform(high = VAR_RANGE,low = -VAR_RANGE,size=(1,DIMENTION)))
+or_bias = (gen.uniform(high = VAR_RANGE, low = -VAR_RANGE,size=()))
+print(or_bias)
+
+input_x = gen.uniform(low = -VAR_RANGE,high=VAR_RANGE,size=(DIMENTION,N))
+y = or_weight @ input_x + or_bias
+error = gen.normal(0,SIGMA,size = (1,N))
+y += error
+print(y)
+
+
+input_var = Variable(np.array([[1],[2],[3]]))
+weight = Variable(np.array([[2,-1,1]]))
+bias = Variable(np.array([[1]]))
+v = ((weight @ input_var) + bias)
+
+print(make_mermaid_graph(v))
+
+print(f"result : {v.numpy()}")
+v.backprop(np.ones(()))
+print(f"grad input : {input_var.grad}, w : {weight.grad}, b : {bias.grad}")