monoid
/
sgd_hw
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

epsilon

This commit is contained in:
monoid 2021-02-20 15:25:40 +09:00
parent 3ec4704618
commit 769ac11912
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
layer.py
View File

@ -82,11 +82,9 @@ class MatMulOp(OpTree):
b = self.b.numpy() if isinstance(self.b,OpTree) else self.b b = self.b.numpy() if isinstance(self.b,OpTree) else self.b
if isinstance(self.a,OpTree): if isinstance(self.a,OpTree):
s = seed * np.transpose(b) if seed.shape == () else (seed) @ np.transpose(b) s = seed * np.transpose(b) if seed.shape == () else (seed) @ np.transpose(b)
#print('seed : ', s)
self.a.backprop((s)) self.a.backprop((s))
if isinstance(self.b,OpTree): if isinstance(self.b,OpTree):
s = np.transpose(a) * seed if seed.shape == () else np.transpose(a) @ seed s = np.transpose(a) * seed if seed.shape == () else np.transpose(a) @ seed
#print('seed : ', s)
self.b.backprop(s) self.b.backprop(s)
def matmul(a,b): def matmul(a,b):
@ -179,18 +177,20 @@ def relu(v):
return FunctionOp(relu_f,relu_diff,"Relu",v) return FunctionOp(relu_f,relu_diff,"Relu",v)
#row vector #row vector
def softmaxHelp(i): def softmaxHelp(i):
e = np.exp(i) m = np.max(i,axis=i.ndim-1)
sumofe = np.sum(e,axis=e.ndim - 1) e = np.exp(i-m.reshape(*m.shape,1))
sumofe = np.sum(e,axis=i.ndim - 1)
sumofe = sumofe.reshape(*sumofe.shape,1) sumofe = sumofe.reshape(*sumofe.shape,1)
return e / sumofe return e / sumofe
class SoftmaxWithNegativeLogLikelihood(OpTree): class SoftmaxWithNegativeLogLikelihood(OpTree):
#row vector #row vector
def __init__(self, i, y): def __init__(self, i, y):
super().__init__() super().__init__()
epsilon = 1e-12
self.i = i self.i = i
self.s = softmaxHelp(i.numpy()) self.s = softmaxHelp(i.numpy())
self.y = y self.y = y
self.v = -y*np.log(self.s) self.v = -y*np.log(self.s+epsilon)
self.v = np.sum(self.v,axis=self.v.ndim-1) self.v = np.sum(self.v,axis=self.v.ndim-1)
def __str__(self): def __str__(self):