import mnist_loader
import network

training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = network.Network([784, 30, 10])
net.SGD(training_data, 30, 10, 3.0, test_data=test_data)

Epoch 0: 8250 / 10000
Epoch 1: 8371 / 10000
Epoch 2: 9300 / 10000
......
Epoch 28: 9552 / 10000
Epoch 29: 9555 / 10000

class Network(object):
    def __init__(self, sizes):self.num_layers = len(sizes)
        self.sizes = sizes
        self.biases = [np.random.randn(y, 1) for y in sizes[1:]]
        self.weights = [np.random.randn(y, x)
                        for x, y in zip(sizes[:-1], sizes[1:])]

def feedforward(self, a):
    """Return the output of the network if ``a`` is input."""for b, w in zip(self.biases, self.weights):
        a = sigmoid(np.dot(w, a)+b)
    return a

#### Miscellaneous functionsdef sigmoid(z):
    """The sigmoid function."""return 1.0/(1.0+np.exp(-z))

def sigmoid_prime(z):
    """Derivative of the sigmoid function."""return sigmoid(z)*(1-sigmoid(z))

net.SGD(training_data, 30, 10, 3.0, test_data=test_data)


def SGD(self, training_data, epochs, mini_batch_size, eta,
        test_data=None):
if test_data: n_test = len(test_data)
    n = len(training_data)
    for j in xrange(epochs):
        random.shuffle(training_data)
        mini_batches = [
            training_data[k:k+mini_batch_size]
            for k in xrange(0, n, mini_batch_size)]
        for mini_batch in mini_batches:
            self.update_mini_batch(mini_batch, eta)
        if test_data:
            print "Epoch {0}: {1} / {2}".format(
                j, self.evaluate(test_data), n_test)
        else:
            print "Epoch {0} complete".format(j)

    for j in xrange(epochs): ## 一共進(jìn)行 epochs=30 輪迭代
        random.shuffle(training_data)  ## 訓(xùn)練數(shù)據(jù)隨機(jī)打散
        mini_batches = [
            training_data[k:k+mini_batch_size]
            for k in xrange(0, n, mini_batch_size)] ## 把50,000個訓(xùn)練數(shù)據(jù)分成5,000個batch，每個batch包含10個訓(xùn)練數(shù)據(jù)。
        for mini_batch in mini_batches: ## 對于每個batch
            self.update_mini_batch(mini_batch, eta) ## 使用梯度下降更新參數(shù)
        if test_data: ## 如果提供了測試數(shù)據(jù)
            print "Epoch {0}: {1} / {2}".format(
                j, self.evaluate(test_data), n_test) ## 評價在測試數(shù)據(jù)上的準(zhǔn)確率
        else:
            print "Epoch {0} complete".format(j)

def evaluate(self, test_data):
    test_results = [(np.argmax(self.feedforward(x)), y)
                    for (x, y) in test_data]
    return sum(int(x == y) for (x, y) in test_results)

def update_mini_batch(self, mini_batch, eta):
nabla_b = [np.zeros(b.shape) for b in self.biases]
  ## 回憶一下__init__，biases是一個列表，包含兩個矩陣，分別是30*1和10*1
  ## 我們先構(gòu)造一個和self.biases一樣大小的列表，用來存放累加的梯度（偏導(dǎo)數(shù)） 
    nabla_w = [np.zeros(w.shape) for w in self.weights]
  ## 同上， weights包含兩個矩陣，大小分別是30*784和10*30 
    for x, y in mini_batch:
        delta_nabla_b, delta_nabla_w = self.backprop(x, y)
     ## 對于一個訓(xùn)練數(shù)據(jù)(x,y)計算loss相對于所有參數(shù)的偏導(dǎo)數(shù)
        ## 因此delta_nabla_b和self.biases， nabla_b是一樣大小(shape)
        ## 同樣delta_nabla_w和self.weights,nabla_w一樣大小
        nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
        ## 把bias的梯度累加到nabla_b里
        nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
        ## 把weight的梯度累加到nable_w里
    self.weights = [w-(eta/len(mini_batch))*nw
                    for w, nw in zip(self.weights, nabla_w)]
     ## 使用這個batch的梯度和eta（步長）更新參數(shù)weights
    self.biases = [b-(eta/len(mini_batch))*nb
                   for b, nb in zip(self.biases, nabla_b)]
        ## 更新biases
        ## 這里更新參數(shù)是除了batch的大小（10），有的人實(shí)現(xiàn)時不除，其實(shí)沒有什么區(qū)別，因為超參數(shù)eta會有所不同，如果不除，那么eta相當(dāng)于是0.3(在eta那里就除了batch的大小了）。

def backprop(self, x, y):
nabla_b = [np.zeros(b.shape) for b in self.biases]
    nabla_w = [np.zeros(w.shape) for w in self.weights]
    # feedforwardactivation = x
    activations = [x] # list to store all the activations, layer by layerzs = [] # list to store all the z vectors, layer by layerfor b, w in zip(self.biases, self.weights):
        z = np.dot(w, activation)+b
        zs.append(z)
        activation = sigmoid(z)
        activations.append(activation)
    # backward passdelta = self.cost_derivative(activations[-1], y) * \
        sigmoid_prime(zs[-1])
    nabla_b[-1] = delta
    nabla_w[-1] = np.dot(delta, activations[-2].transpose())
for l in xrange(2, self.num_layers):
        z = zs[-l]
        sp = sigmoid_prime(z)
        delta = np.dot(self.weights[-l+1].transpose(), delta) * sp
        nabla_b[-l] = delta
        nabla_w[-l] = np.dot(delta, activations[-l-1].transpose())
    return (nabla_b, nabla_w)