Implementing in PyTorch

In [1]:

import math

import math

In [2]:

class Value:

    def __init__(self, data, _children=(), _op='', label=''):
        self.data = data
        self.grad = 0.0
        self._backward = lambda: None #Its an empty function by default. This is what will do that gradient calculation at each of the operations.
        self._prev = set(_children)
        self._op = _op
        self.label = label


    def __repr__(self):
        return f"Value(data={self.data})"

    def __add__(self, other):
        other = other if isinstance(other, Value) else Value(other)
        out = Value(self.data + other.data, (self, other), '+')

        def backward():
          self.grad += 1.0 * out.grad
          other.grad += 1.0 * out.grad

        out._backward = backward
        return out

    def __mul__(self, other):
        other = other if isinstance(other, Value) else Value(other)
        out = Value(self.data * other.data, (self, other), '*')

        def backward():
          self.grad += other.data * out.grad
          other.grad += self.data * out.grad
        out._backward = backward
        return out

    def __rmul__(self, other):   #other * self
        return self * other

    def __truediv__(self, other):  #self/other
        return self * other**-1

    def __neg__(self):
        return self * -1

    def __sub__(self, other):  #self - other
        return self + (-other)

    def __pow__(self, other):
        assert isinstance(other, (int, float)), "only supporting int/float powers for now"
        out = Value(self.data ** other, (self, ), f"**{other}")

        def backward():
          self.grad += (other * (self.data ** (other - 1))) * out.grad

        out._backward = backward
        return out

    def tanh(self):
        x = self.data
        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)
        out = Value(t, (self, ), 'tanh')

        def backward():
          self.grad += 1 - (t**2) * out.grad

        out._backward = backward
        return out

    def exp(self):
        x = self.data
        out = Value(math.exp(x), (self, ), 'exp')   #We merged t and out, into just out

        def backward():
            self.grad += out.data * out.grad

        out._backward = backward
        return out

    def backward(self):

      topo = []
      visited = set()
      def build_topo(v):
        if v not in visited:
          visited.add(v)
          for child in v._prev:
            build_topo(child)
          topo.append(v)

      build_topo(self)

      self.grad = 1.0
      for node in reversed(topo):
        node._backward()

class Value: def __init__(self, data, _children=(), _op='', label=''): self.data = data self.grad = 0.0 self._backward = lambda: None #Its an empty function by default. This is what will do that gradient calculation at each of the operations. self._prev = set(_children) self._op = _op self.label = label def __repr__(self): return f"Value(data={self.data})" def __add__(self, other): other = other if isinstance(other, Value) else Value(other) out = Value(self.data + other.data, (self, other), '+') def backward(): self.grad += 1.0 * out.grad other.grad += 1.0 * out.grad out._backward = backward return out def __mul__(self, other): other = other if isinstance(other, Value) else Value(other) out = Value(self.data * other.data, (self, other), '*') def backward(): self.grad += other.data * out.grad other.grad += self.data * out.grad out._backward = backward return out def __rmul__(self, other): #other * self return self * other def __truediv__(self, other): #self/other return self * other**-1 def __neg__(self): return self * -1 def __sub__(self, other): #self - other return self + (-other) def __pow__(self, other): assert isinstance(other, (int, float)), "only supporting int/float powers for now" out = Value(self.data ** other, (self, ), f"**{other}") def backward(): self.grad += (other * (self.data ** (other - 1))) * out.grad out._backward = backward return out def tanh(self): x = self.data t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1) out = Value(t, (self, ), 'tanh') def backward(): self.grad += 1 - (t**2) * out.grad out._backward = backward return out def exp(self): x = self.data out = Value(math.exp(x), (self, ), 'exp') #We merged t and out, into just out def backward(): self.grad += out.data * out.grad out._backward = backward return out def backward(self): topo = [] visited = set() def build_topo(v): if v not in visited: visited.add(v) for child in v._prev: build_topo(child) topo.append(v) build_topo(self) self.grad = 1.0 for node in reversed(topo): node._backward()

In [ ]:

x1 = Value(2.0, label='x1')
x2 = Value(0.0, label='x2')

w1 = Value(-3.0, label='w1')
w2 = Value(1.0, label='w2')

b = Value(6.8813735870195432, label='b')

x1w1 = x1*w1; x1w1.label = 'x1*w1'
x2w2 = x2*w2; x2w2.label = 'x2*w2'

x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2'

n = x1w1x2w2 + b; n.label = 'n'

#o = n.tanh(); o.label = 'o'

#Spliting up of the tanh function

e = (2*n).exp()
o = (e - 1) / (e + 1)
o.label = 'o'

x1 = Value(2.0, label='x1') x2 = Value(0.0, label='x2') w1 = Value(-3.0, label='w1') w2 = Value(1.0, label='w2') b = Value(6.8813735870195432, label='b') x1w1 = x1*w1; x1w1.label = 'x1*w1' x2w2 = x2*w2; x2w2.label = 'x2*w2' x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2' n = x1w1x2w2 + b; n.label = 'n' #o = n.tanh(); o.label = 'o' #Spliting up of the tanh function e = (2*n).exp() o = (e - 1) / (e + 1) o.label = 'o'

---

Now we'll how the same thing can be written in PyTorch syntax

In [3]:

import torch

import torch

In [4]:

x1 = torch.Tensor([2.0]).double() ; x1.requires_grad = True
x2 = torch.Tensor([0.0]).double() ; x2.requires_grad = True
w1 = torch.Tensor([-3.0]).double() ; w1.requires_grad = True
w2 = torch.Tensor([1.0]).double() ; w2.requires_grad = True
b = torch.Tensor([6.8813735870195432]).double() ; b.requires_grad = True

n = x1*w1 + x2*w2 + b
o = torch.tanh(n)

print(o.data.item())
o.backward()

print('x2', x2.grad.item())
print('w2', w2.grad.item())
print('x1', x1.grad.item())
print('w1', w1.grad.item())

x1 = torch.Tensor([2.0]).double() ; x1.requires_grad = True x2 = torch.Tensor([0.0]).double() ; x2.requires_grad = True w1 = torch.Tensor([-3.0]).double() ; w1.requires_grad = True w2 = torch.Tensor([1.0]).double() ; w2.requires_grad = True b = torch.Tensor([6.8813735870195432]).double() ; b.requires_grad = True n = x1*w1 + x2*w2 + b o = torch.tanh(n) print(o.data.item()) o.backward() print('x2', x2.grad.item()) print('w2', w2.grad.item()) print('x1', x1.grad.item()) print('w1', w1.grad.item())

0.7071066904050358
x2 0.5000001283844369
w2 0.0
x1 -1.5000003851533106
w1 1.0000002567688737

---

Some PyTorch syntax review

In [5]:

torch.Tensor([[1,2,3], [4,5,6]])

torch.Tensor([[1,2,3], [4,5,6]])

Out[5]:

tensor([[1., 2., 3.],
        [4., 5., 6.]])

In [6]:

torch.Tensor([[1,2,3], [4,5,6]]).shape

torch.Tensor([[1,2,3], [4,5,6]]).shape

Out[6]:

torch.Size([2, 3])

In [7]:

torch.Tensor([2.0]).dtype

torch.Tensor([2.0]).dtype

Out[7]:

torch.float32

In [8]:

torch.Tensor([2.0]).double().dtype

torch.Tensor([2.0]).double().dtype

Out[8]:

torch.float64

In [9]:

x2.data

x2.data

Out[9]:

tensor([0.], dtype=torch.float64)

In [10]:

x2.data.item()

x2.data.item()

Out[10]:

0.0

---