Optimization Single Step Preview

In [1]:

class Value:

    def __init__(self, data, _children=(), _op='', label=''):
        self.data = data
        self.grad = 0.0
        self._prev = set(_children)
        self._op = _op
        self.label = label


    def __repr__(self):   # This basically allows us to print nicer looking expressions for the final output
        return f"Value(data={self.data})"

    def __add__(self, other):
        out = Value(self.data + other.data, (self, other), '+')
        return out

    def __mul__(self, other):
        out = Value(self.data * other.data, (self, other), '*')
        return out

class Value: def __init__(self, data, _children=(), _op='', label=''): self.data = data self.grad = 0.0 self._prev = set(_children) self._op = _op self.label = label def __repr__(self): # This basically allows us to print nicer looking expressions for the final output return f"Value(data={self.data})" def __add__(self, other): out = Value(self.data + other.data, (self, other), '+') return out def __mul__(self, other): out = Value(self.data * other.data, (self, other), '*') return out

In [2]:

a = Value(2.0, label='a')
b = Value(-3.0, label='b')
c = Value(10.0, label='c')
e = a*b; e.label='e'
d= e + c; d.label='d'
f = Value(-2.0, label='f')
L = d*f; L.label='L'
L

a = Value(2.0, label='a') b = Value(-3.0, label='b') c = Value(10.0, label='c') e = a*b; e.label='e' d= e + c; d.label='d' f = Value(-2.0, label='f') L = d*f; L.label='L' L

Out[2]:

Value(data=-8.0)

In [3]:

from graphviz import Digraph

def trace(root):
    #Builds a set of all nodes and edges in a graph
    nodes, edges = set(), set()
    def build(v):
        if v not in nodes:
            nodes.add(v)
            for child in v._prev:
                edges.add((child, v))
                build(child)
    build(root)
    return nodes, edges

def draw_dot(root):
    dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'}) #LR == Left to Right

    nodes, edges = trace(root)
    for n in nodes:
        uid = str(id(n))
        #For any value in the graph, create a rectangular ('record') node for it
        dot.node(name = uid, label = "{ %s | data %.4f | grad %.4f }" % ( n.label, n.data, n.grad), shape='record')
        if n._op:
            #If this value is a result of some operation, then create an op node for it
            dot.node(name = uid + n._op, label=n._op)
            #and connect this node to it
            dot.edge(uid + n._op, uid)

    for n1, n2 in edges:
        #Connect n1 to the node of n2
        dot.edge(str(id(n1)), str(id(n2)) + n2._op)

    return dot

from graphviz import Digraph def trace(root): #Builds a set of all nodes and edges in a graph nodes, edges = set(), set() def build(v): if v not in nodes: nodes.add(v) for child in v._prev: edges.add((child, v)) build(child) build(root) return nodes, edges def draw_dot(root): dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'}) #LR == Left to Right nodes, edges = trace(root) for n in nodes: uid = str(id(n)) #For any value in the graph, create a rectangular ('record') node for it dot.node(name = uid, label = "{ %s | data %.4f | grad %.4f }" % ( n.label, n.data, n.grad), shape='record') if n._op: #If this value is a result of some operation, then create an op node for it dot.node(name = uid + n._op, label=n._op) #and connect this node to it dot.edge(uid + n._op, uid) for n1, n2 in edges: #Connect n1 to the node of n2 dot.edge(str(id(n1)), str(id(n2)) + n2._op) return dot

In [4]:

L.grad = 1.0
f.grad = 4.0
d.grad = -2.0
c.grad = -2.0
e.grad = -2.0
a.grad = 6.0
b.grad = -4.0

L.grad = 1.0 f.grad = 4.0 d.grad = -2.0 c.grad = -2.0 e.grad = -2.0 a.grad = 6.0 b.grad = -4.0

In [5]:

draw_dot(L)

draw_dot(L)

Out[5]:

---

Now, we are going to try and nudge the leaf nodes (because those are usually what we have control over. In our example: a,b,c,f) slightly towards the gradient value, to nudge L towards a more positive direction.

In [7]:

a.data += 0.01 * a.grad
b.data += 0.01 * b.grad
c.data += 0.01 * c.grad
f.data += 0.01 * f.grad

e = a*b;
d= e + c;
L = d*f;
L

print(L.data)

a.data += 0.01 * a.grad b.data += 0.01 * b.grad c.data += 0.01 * c.grad f.data += 0.01 * f.grad e = a*b; d= e + c; L = d*f; L print(L.data)

-6.723584000000001

Therefore the value of L was pushed to a more positive direction from -8.0 to -6.0