This commit is contained in:
monoid 2021-02-12 15:50:20 +09:00
parent 92ad16ecbb
commit 34bf851d1a
6 changed files with 228 additions and 0 deletions

1
.gitignore vendored
View File

@ -138,3 +138,4 @@ dmypy.json
# Cython debug symbols # Cython debug symbols
cython_debug/ cython_debug/
sgd_hw/

3
.vscode/settings.json vendored Normal file
View File

@ -0,0 +1,3 @@
{
"python.pythonPath": "sgd_hw\\Scripts\\python.exe"
}

BIN
AI_2020_project_2.pdf Normal file

Binary file not shown.

192
layer.py Normal file
View File

@ -0,0 +1,192 @@
import numpy as np
from contextlib import contextmanager
from typing import Generator, Dict, Union
import io
#only scalar gradient
#op must be tree. 그래프 구현할려면, 위상정렬해서 순회해야하기 때문에 그렇게 하지 않음.
class NonExistVarableError(ValueError):
pass
def make_mermaid_graph(result):
with io.StringIO("") as graph:
graph.write("graph TD\n")
result.mermaid_graph(graph)
graph.write(f"{id(result)}-->Result\n")
return graph.getvalue()
class OpTree:
def __init__(self):
super().__init__()
def __matmul__(self,a):
return MatMulOp(self,a)
def __add__(self,a):
return AddOp(self,a)
@property
def T(self):
return TransposeOp(self)
class MatMulOp(OpTree):
def __init__(self,a,b):
super().__init__()
self.a = a
self.b = b
va = self.a.numpy() if isinstance(self.a,OpTree) else self.a
vb = self.b.numpy() if isinstance(self.b,OpTree) else self.b
self.v = va @ vb
def __str__(self):
return f"MatmulOp"
def mermaid_graph(self,writer):
if isinstance(self.a,OpTree):
self.a.mermaid_graph(writer)
writer.write(f'{id(self.a)}-->{id(self)}[MatmulOp]\n')
if isinstance(self.b,OpTree):
self.b.mermaid_graph(writer)
writer.write(f'{id(self.b)}-->{id(self)}[MatmulOp]\n')
def numpy(self):
return self.v
def backprop(self,seed):
#a @ b
a = self.a.numpy() if isinstance(self.a,OpTree) else self.a
b = self.b.numpy() if isinstance(self.b,OpTree) else self.b
if isinstance(self.a,OpTree):
s = seed * np.transpose(b) if seed.shape == () else (seed) @ np.transpose(b)
#print('seed : ', s)
self.a.backprop((s))
if isinstance(self.b,OpTree):
s = np.transpose(a) * seed if seed.shape == () else np.transpose(a) @ seed
#print('seed : ', s)
self.b.backprop(s)
def matmul(a,b):
return MatMulOp(a,b)
class AddOp(OpTree):
def __init__(self,a,b):
super().__init__()
self.a = a
self.b = b
va = self.a.numpy() if isinstance(self.a,OpTree) else self.a
vb = self.b.numpy() if isinstance(self.b,OpTree) else self.b
self.v = va + vb
def __str__(self):
return f"AddOp"
def mermaid_graph(self,writer):
if isinstance(self.a,OpTree):
self.a.mermaid_graph(writer)
writer.write(f'{id(self.a)}-->{id(self)}[AddOp]\n')
if isinstance(self.b,OpTree):
self.b.mermaid_graph(writer)
writer.write(f'{id(self.b)}-->{id(self)}[AddOp]\n')
def numpy(self):
return self.v
def backprop(self,seed):
#borad_casted = self.a.shape != self.b.shape
#np.ones((1,b.shape[1]))
#a + b
if isinstance(self.a,OpTree):
self.a.backprop(seed)
if isinstance(self.b,OpTree):
self.b.backprop(seed)
def addmul(a,b):
return AddOp(a,b)
class FunctionOp(OpTree):
def __init__(self,f, f_grad, f_name, i):
super().__init__()
self.f = np.vectorize(f)
self.f_grad = np.vectorize(f_grad)
self.f_name = f_name
self.i = i
self.v = self.f(i.numpy())
def __str__(self):
return f"Function{self.f_name}Op"
def mermaid_graph(self,writer):
self.i.mermaid_graph(writer)
writer.write(f'{id(self.i)}-->{id(self)}[Function{self.f_name}Op]\n')
def numpy(self):
return self.v
def backprop(self,seed):
self.i.backprop(seed * (self.f_grad(self.i.numpy())))
class TransposeOp(OpTree):
def __init__(self, i):
super().__init__()
self.i = i
self.v = np.transpose(i.numpy())
def __str__(self):
return f"TransposeOp"
def mermaid_graph(self,writer):
self.i.mermaid_graph(writer)
writer.write(f'{id(self.i)}-->{id(self)}[TransposeOp]\n')
def numpy(self):
return self.v
def backprop(self,seed):
self.i.backprop(np.transpose(seed))
def transposemul(a):
return TransposeOp(a)
def relu(v):
relu_f = lambda x: np.max([x,0])
relu_diff = lambda x: 1 if x > 0 else 0
return FunctionOp(relu_f,relu_diff,"Relu",v)
class Variable(OpTree):
def __init__(self,x):
super().__init__()
self.x = x
self.grad = None
def numpy(self):
return self.x
def mermaid_graph(self,writer):
writer.write(f'{id(self)}["Variable{self.x.shape}"]\n')
def backprop(self,seed):
self.grad = seed
"""
input_var = Variable(np.array([[1],[2],[3]]))
weight = Variable(np.array([[2,-1,1]]))
v = relu(weight @ input_var)
print(f"result : {v.numpy()}")
v.backprop(np.ones(()))
print(f"grad input : {input_var.grad}, w : {weight.grad}")
"""
#input_diff = Variable(np.array([[1.01],[2],[3]]))
#v_diff = relu(weight @ input_diff)
#print(f"diff 1 : {(np.sum(v_diff.numpy()) - v.numpy()) / 0.01}")
#i -= grad * delta
"""
graph TD
2284612545696["Variable(1, 3)"]
2284612545696-->2284612624880[MatmulOp]
2284612544496["Variable(3, 2)"]
2284612544496-->2284612624880[MatmulOp]
2284612624880-->2284612625072[FunctionReluOp]
2284612625072-->2284612627856[MatmulOp]
2284612627856-->Result
"""

32
p2.py Normal file
View File

@ -0,0 +1,32 @@
from layer import *
import numpy as np
import pickle
DIMENTION = 3
VAR_RANGE = 1
N = 10
ALPHA = 0.1
SIGMA = VAR_RANGE * ALPHA
gen: np.random.Generator = np.random.default_rng()
or_weight = (gen.uniform(high = VAR_RANGE,low = -VAR_RANGE,size=(1,DIMENTION)))
or_bias = (gen.uniform(high = VAR_RANGE, low = -VAR_RANGE,size=()))
print(or_bias)
input_x = gen.uniform(low = -VAR_RANGE,high=VAR_RANGE,size=(DIMENTION,N))
y = or_weight @ input_x + or_bias
error = gen.normal(0,SIGMA,size = (1,N))
y += error
print(y)
input_var = Variable(np.array([[1],[2],[3]]))
weight = Variable(np.array([[2,-1,1]]))
bias = Variable(np.array([[1]]))
v = ((weight @ input_var) + bias)
print(make_mermaid_graph(v))
print(f"result : {v.numpy()}")
v.backprop(np.ones(()))
print(f"grad input : {input_var.grad}, w : {weight.grad}, b : {bias.grad}")

BIN
requirements.txt Normal file

Binary file not shown.