ML_Softmax_Regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------- do_train: start -----------------\n",
      "file_path =  ['/home/yangyuqi/work/tf_algorithm_example/data_source/iris.csv'] \n",
      "\n",
      "Tensor(\"ReaderRead:0\", shape=(), dtype=string) Tensor(\"ReaderRead:1\", shape=(), dtype=string)\n",
      "in train, loss:  Tensor(\"Mean:0\", shape=(), dtype=float32)\n",
      "in train, _W:  <tensorflow.python.ops.variables.Variable object at 0x7f0a68bbb710>\n",
      "in train, _b:  <tensorflow.python.ops.variables.Variable object at 0x7f0a65c99dd8>\n",
      "features_0 tensor.shape = [10  4], tensor = [[ 4.80000019  3.4000001   1.60000002  0.2       ]\n",
      " [ 5.19999981  3.5         1.5         0.2       ]\n",
      " [ 4.80000019  3.0999999   1.60000002  0.2       ]\n",
      " [ 4.80000019  3.4000001   1.89999998  0.2       ]\n",
      " [ 4.69999981  3.20000005  1.60000002  0.2       ]\n",
      " [ 4.9000001   3.0999999   1.5         0.1       ]\n",
      " [ 5.0999999   3.5         1.39999998  0.30000001]\n",
      " [ 5.          3.20000005  1.20000005  0.2       ]\n",
      " [ 5.5         4.19999981  1.39999998  0.2       ]\n",
      " [ 5.4000001   3.4000001   1.70000005  0.2       ]]\n",
      "\n",
      "label_0 tensor.shape = [10], tensor = [1 0 0 1 0 0 1 0 0 0]\n",
      "\n",
      "step_0 loss:  tensor.shape = [], tensor = 1.2869031429290771\n",
      "\n",
      "training end. step_9 final loss:  tensor.shape = [], tensor = 1.3581855297088623\n",
      "\n",
      "save_path is:  /tmp/ML_Softmax_Regression.vari\n",
      "At evaluate, the _W tensor.shape = [4 3], tensor = [[-0.02271985 -0.04231079  0.06503064]\n",
      " [ 0.0068605  -0.02298325  0.01612274]\n",
      " [-0.05811652 -0.03115496  0.08927147]\n",
      " [-0.02417543 -0.0127696   0.03694503]]\n",
      "\n",
      "At evaluate, the _b tensor.shape = [3], tensor = [-0.00070478 -0.00569872  0.0064035 ]\n",
      "\n",
      "At evaluate, test_features tensor.shape = [10  4], tensor = [[ 5.          3.29999995  1.39999998  0.2       ]\n",
      " [ 4.9000001   2.4000001   3.29999995  1.        ]\n",
      " [ 5.0999999   3.29999995  1.70000005  0.5       ]\n",
      " [ 6.9000001   3.0999999   4.9000001   1.5       ]\n",
      " [ 6.69999981  3.0999999   4.4000001   1.39999998]\n",
      " [ 6.30000019  3.29999995  4.69999981  1.60000002]\n",
      " [ 5.0999999   3.79999995  1.89999998  0.40000001]\n",
      " [ 5.0999999   3.79999995  1.60000002  0.2       ]\n",
      " [ 4.5         2.29999995  1.29999995  0.30000001]\n",
      " [ 4.4000001   3.          1.29999995  0.2       ]]\n",
      "\n",
      "At evaluate, test_label tensor.shape = [10], tensor = [0 1 0 0 1 1 1 1 1 1]\n",
      "\n",
      "evaluate_result tensor.shape = [], tensor = 0.0\n",
      "\n",
      "----------------- do_train: finish -----------------\n",
      "-------------- do_evaluate: start -----------------\n",
      "\n",
      "file_path =  ['/home/yangyuqi/work/tf_algorithm_example/data_source/iris.csv'] \n",
      "\n",
      "Tensor(\"ReaderRead_1:0\", shape=(), dtype=string) Tensor(\"ReaderRead_1:1\", shape=(), dtype=string)\n",
      "At evaluate, the _W tensor.shape = [4 3], tensor = [[-0.02271985 -0.04231079  0.06503064]\n",
      " [ 0.0068605  -0.02298325  0.01612274]\n",
      " [-0.05811652 -0.03115496  0.08927147]\n",
      " [-0.02417543 -0.0127696   0.03694503]]\n",
      "\n",
      "At evaluate, the _b tensor.shape = [3], tensor = [-0.00070478 -0.00569872  0.0064035 ]\n",
      "\n",
      "At evaluate, test_features tensor.shape = [10  4], tensor = [[ 5.          3.4000001   1.60000002  0.40000001]\n",
      " [ 4.69999981  3.20000005  1.29999995  0.2       ]\n",
      " [ 5.4000001   3.4000001   1.5         0.40000001]\n",
      " [ 5.19999981  3.5         1.5         0.2       ]\n",
      " [ 4.80000019  3.0999999   1.60000002  0.2       ]\n",
      " [ 5.0999999   3.70000005  1.5         0.40000001]\n",
      " [ 5.          3.4000001   1.5         0.2       ]\n",
      " [ 5.          3.20000005  1.20000005  0.2       ]\n",
      " [ 5.5         4.19999981  1.39999998  0.2       ]\n",
      " [ 5.0999999   3.4000001   1.5         0.2       ]]\n",
      "\n",
      "At evaluate, test_label tensor.shape = [10], tensor = [0 0 0 1 1 1 0 0 1 0]\n",
      "\n",
      "accuracy_rate tensor.shape = [], tensor = 0.0\n",
      "\n",
      "----------------- do_evaluate: finish -----------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "######################## 机器学习对数几率回归类 ##############################\n",
    "# Author: 杨玉奇\n",
    "# email: yangyuqi@sina.com\n",
    "# url: https://github.com/jerryyyq/tf_algorithm_example\n",
    "# copyright yangyuqi\n",
    "# 著作权归作者 杨玉奇 所有。商业转载请联系作者获得授权，非商业转载请注明出处。\n",
    "# date: 2017-09-19\n",
    "###################################################################\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "from common import *\n",
    "from ML_Model import ML_Model\n",
    "\n",
    "    \n",
    "class ML_Softmax_Regression( ML_Model ):\n",
    "    def __init__(self, feature_number, label_number):\n",
    "        ML_Model.__init__(self)\n",
    "\n",
    "        self._W = tf.Variable(tf.zeros([feature_number, label_number]), name = 'weights')\n",
    "        self._b = tf.Variable(tf.zeros([label_number]), name = 'bias')\n",
    "    \n",
    "        \n",
    "    def combine_inputs(self, features):\n",
    "        return tf.matmul(features, self._W) + self._b\n",
    "\n",
    "    def inference(self, features):\n",
    "        return tf.nn.softmax( self.combine_inputs(features) )\n",
    "    \n",
    "    def loss(self, features, label):\n",
    "        label_predicted = self.combine_inputs(features)\n",
    "        return tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits(logits = label_predicted, labels = label) )\n",
    "    \n",
    "    \n",
    "    def inputs(self, file_name = [], batch_size = 10):\n",
    "        sepal_length, sepal_width, setal_length, setal_width, label = \\\n",
    "        read_csv(batch_size, file_name, [[0.0], [0.0], [0.0], [0.0], ['']])\n",
    "\n",
    "        # return passenger_id, survived \n",
    "        label_number = tf.to_int32(tf.argmax(tf.to_int32(tf.pack([ # 0.9 use tf.pack\n",
    "        # label_number = tf.to_int32(tf.argmax(tf.to_int32(tf.stack([ # 1.3 use tf.stack\n",
    "            tf.equal(label, ['Iris-setosa']),\n",
    "            tf.equal(label, ['Iris-versicolor']),\n",
    "            tf.equal(label, ['Iris-virginica'])\n",
    "        ])), 0))\n",
    "        \n",
    "        features = tf.transpose(tf.pack(\n",
    "        # features = tf.transpose(tf.stack( # 1.3 use tf.stack\n",
    "                [sepal_length, sepal_width, setal_length, setal_width]\n",
    "        ))\n",
    "\n",
    "        return features, label_number \n",
    "        \n",
    "        \n",
    "    def train(self, features, label):\n",
    "        self._total_loss = self.loss( features, label )\n",
    "        print( 'in train, loss: ', self._total_loss )\n",
    "        print( 'in train, _W: ', self._W )\n",
    "        print( 'in train, _b: ', self._b )\n",
    "\n",
    "        learning_rate = 0.01\n",
    "        return tf.train.GradientDescentOptimizer( learning_rate ).minimize( self._total_loss )\n",
    "\n",
    "    \n",
    "    def evaluate(self, test_features, test_label):\n",
    "        echo_tensor(self._sess, self._W, 'At evaluate, the _W')\n",
    "        echo_tensor(self._sess, self._b, 'At evaluate, the _b')\n",
    "        echo_tensor(self._sess, test_features, 'At evaluate, test_features')\n",
    "        echo_tensor(self._sess, test_label, 'At evaluate, test_label')\n",
    "        \n",
    "        label_predicted = tf.cast( tf.arg_max(self.inference(test_features), 1), tf.int32 )\n",
    "        \n",
    "        evaluate_result = tf.reduce_mean(tf.cast(tf.equal(label_predicted, test_label), tf.float32))\n",
    "        evaluate_result = tf.Print( evaluate_result, [evaluate_result], '*********** evaluate_result: ' )\n",
    "        return evaluate_result\n",
    "\n",
    "    \n",
    "if __name__ == '__main__':\n",
    "    one_ml = ML_Softmax_Regression(4, 3)\n",
    "    # data from: https://archive.ics.uci.edu/ml/datasets/Iris\n",
    "    one_ml.do_train( 10, ['data_source/iris.csv'] )  # 10000 次可以得到 80% 的准确率\n",
    "\n",
    "    one_ml.do_evaluate( ['data_source/iris.csv'] )\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------- input value -----------\n",
      "\n",
      "features_pack:  [[ 5.          7.          5.69999981  6.30000019]\n",
      " [ 3.29999995  3.20000005  2.79999995  3.29999995]\n",
      " [ 1.39999998  4.69999981  4.0999999   6.        ]\n",
      " [ 0.2         1.39999998  1.29999995  2.5       ]]\n",
      "features:  [[ 5.          3.29999995  1.39999998  0.2       ]\n",
      " [ 7.          3.20000005  4.69999981  1.39999998]\n",
      " [ 5.69999981  2.79999995  4.0999999   1.29999995]\n",
      " [ 6.30000019  3.29999995  6.          2.5       ]] \n",
      "\n",
      "label:  [[ True False False False]\n",
      " [False  True  True False]\n",
      " [False False False  True]] \n",
      "\n",
      " [[1 0 0 0]\n",
      " [0 1 1 0]\n",
      " [0 0 0 1]]\n",
      "label_max:  [0 1 1 2]\n",
      "label_number:  [0 1 1 2]\n",
      "\n",
      "---------- traning -----------\n",
      "\n",
      "W:  [[0.87548745, 0.80772996, -1.68321049], [2.02138662, -0.17534477, -1.84603953], [-2.75662613, -0.15486683, 2.91150331], [-1.28630328, -1.03182793, 2.31813335]]\n",
      "b:  [0.41992012, 0.57542503, -0.99534476]\n",
      "label_combine:  [[  7.35139561   3.61225796 -10.9635973 ]\n",
      " [ -1.74019706   3.49599838  -1.75569403]\n",
      " [ -1.90428054   2.71219015  -0.80781806]\n",
      " [ -7.14944935   1.57671535   5.57285452]]\n",
      "label_softmax:  [ 0.02349641  0.01050372  0.0387271   0.01822242]\n",
      "label_loss:  0.0227374\n",
      "\n",
      "---------- evaluate -----------\n",
      "\n",
      "label_inference:  [[  9.76777494e-01   2.32224911e-02   1.08566356e-08]\n",
      " [  5.26486756e-03   9.89551246e-01   5.18390862e-03]\n",
      " [  9.51203052e-03   9.62013185e-01   2.84748171e-02]\n",
      " [  2.92994810e-06   1.80544779e-02   9.81942594e-01]]\n",
      "label_arg_max:  [0 1 1 2]\n",
      "label_predicted:  [0 1 1 2]\n",
      "evaluate:  [ 1.  1.  1.  1.]\n",
      "accuracy_rate:  1.0\n"
     ]
    }
   ],
   "source": [
    "# 中间计算数据样例展示\n",
    "import tensorflow as tf\n",
    "\n",
    "'''\n",
    "5.0,3.3,1.4,0.2,Iris-setosa\n",
    "7.0,3.2,4.7,1.4,Iris-versicolor\n",
    "5.7,2.8,4.1,1.3,Iris-versicolor\n",
    "6.3,3.3,6.0,2.5,Iris-virginica\n",
    "'''\n",
    "sepal_length = [5.0, 7.0, 5.7, 6.3]\n",
    "sepal_width = [3.3, 3.2, 2.8, 3.3]\n",
    "setal_length = [1.4, 4.7, 4.1, 6.0]\n",
    "setal_width = [0.2, 1.4, 1.3, 2.5]\n",
    "\n",
    "features_pack = tf.pack([sepal_length, sepal_width, setal_length, setal_width])\n",
    "features = tf.transpose( features_pack )\n",
    "\n",
    "\n",
    "label = ['Iris-setosa', 'Iris-versicolor', 'Iris-versicolor', 'Iris-virginica']\n",
    "\n",
    "label_pack = tf.pack([\n",
    "            tf.equal(label, ['Iris-setosa']),\n",
    "            tf.equal(label, ['Iris-versicolor']),\n",
    "            tf.equal(label, ['Iris-virginica'])\n",
    "        ])\n",
    "\n",
    "label_pack_int = tf.to_int32( label_pack )\n",
    "label_pack_max = tf.argmax( label_pack_int, 0 )\n",
    "test_label = tf.to_int32( label_pack_max )\n",
    "\n",
    "''' 学习 10 次获得的值\n",
    "W = [[-0.02240583, -0.04530681, 0.06771266],\n",
    " [ 0.00781024, -0.02450576, 0.01669552],\n",
    " [-0.05717348, -0.03503608, 0.09220956],\n",
    " [-0.02351422, -0.01420259, 0.03771682]]\n",
    "\n",
    "b = [-0.00052203, -0.00652796, 0.00704999]\n",
    "'''\n",
    "\n",
    "# 学习 10000 次获得的值\n",
    "W = [[0.87548745, 0.80772996, -1.68321049],\n",
    " [2.02138662, -0.17534477, -1.84603953],\n",
    " [-2.75662613, -0.15486683, 2.91150331],\n",
    " [-1.28630328, -1.03182793, 2.31813335]]\n",
    "\n",
    "b = [0.41992012, 0.57542503, -0.99534476]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    print( '---------- input value -----------\\n' )\n",
    "    print( 'features_pack: ', sess.run(features_pack) )\n",
    "    print( 'features: ', sess.run( features ), '\\n' )\n",
    "    \n",
    "    print( 'label: ', sess.run( label_pack ), '\\n\\n', sess.run(label_pack_int) )\n",
    "    print( 'label_max: ', sess.run( label_pack_max ) )\n",
    "    print( 'label_number: ', sess.run( label_number ) )\n",
    "    \n",
    "    print( '\\n---------- traning -----------\\n' )\n",
    "    label_combine = tf.matmul( features, W ) + b\n",
    "    label_softmax = tf.nn.sparse_softmax_cross_entropy_with_logits( label_combine, test_label )\n",
    "    label_loss = tf.reduce_mean( label_softmax )\n",
    "    \n",
    "    print( 'W: ', W )\n",
    "    print( 'b: ', b )    \n",
    "    print( 'label_combine: ', sess.run(label_combine) )\n",
    "    print( 'label_softmax: ', sess.run(label_softmax) )\n",
    "    print( 'label_loss: ', sess.run(label_loss) )\n",
    "    \n",
    "    print( '\\n---------- evaluate -----------\\n' )\n",
    "    label_inference = tf.nn.softmax( label_combine )\n",
    "    label_arg_max = tf.arg_max( label_inference, 1 )\n",
    "    label_predicted = tf.cast( label_arg_max, tf.int32 )\n",
    "   \n",
    "    print( 'label_inference: ', sess.run(label_inference) )\n",
    "    print( 'label_arg_max: ', sess.run(label_arg_max) )\n",
    "    print( 'label_predicted: ', sess.run(label_predicted) )\n",
    "    \n",
    "    evaluate = tf.cast( tf.equal( label_predicted, test_label ), tf.float32 )\n",
    "    accuracy_rate = tf.reduce_mean( evaluate )\n",
    "\n",
    "    print( 'evaluate: ', sess.run(evaluate) )\n",
    "    print( 'accuracy_rate: ', sess.run(accuracy_rate) )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---------- traning -----------\n",
    "\n",
    "W:  [[-0.02240583, -0.04530681, 0.06771266], [0.00781024, -0.02450576, 0.01669552], [-0.05717348, -0.03503608, 0.09220956], [-0.02351422, -0.01420259, 0.03771682]]\n",
    "b:  [-0.00052203, -0.00652796, 0.00704999]\n",
    "label_combine:  [[-0.17152311 -0.36582205  0.53734523]\n",
    " [-0.43400532 -0.58664727  1.02065277]\n",
    " [-0.37134629 -0.49550417  0.86685061]\n",
    " [-0.51773143 -0.61855286  1.13628435]]\n",
    "label_softmax:  [ 1.34939766  1.96770382  1.79800224  0.31057963]\n",
    "label_loss:  1.35642\n",
    "\n",
    "---------- evaluate -----------\n",
    "\n",
    "label_inference:  [[ 0.25939646  0.2135901   0.52701342]\n",
    " [ 0.16282785  0.13977745  0.69739473]\n",
    " [ 0.1875248   0.16562946  0.64684576]\n",
    " [ 0.1402126   0.12676543  0.73302197]]\n",
    "label_arg_max:  [2 2 2 2]\n",
    "label_predicted:  [2 2 2 2]\n",
    "evaluate:  [ 0.  0.  0.  1.]\n",
    "accuracy_rate:  0.25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__train_label:  [[0 0 0 ..., 0 0 0]\n",
      " [1 0 0 ..., 0 0 0]\n",
      " [0 0 0 ..., 0 0 0]\n",
      " ..., \n",
      " [0 0 0 ..., 0 0 0]\n",
      " [0 0 0 ..., 0 0 0]\n",
      " [0 0 0 ..., 0 1 0]]\n",
      "-------------- do_train: start -----------------\n",
      "in train, loss:  Tensor(\"Mean_7:0\", shape=(), dtype=float32)\n",
      "in train, _W:  <tensorflow.python.ops.variables.Variable object at 0x7ffa7c0f64a8>\n",
      "in train, _b:  <tensorflow.python.ops.variables.Variable object at 0x7ffa7c0f6710>\n",
      "features_0 not_tensor = [[ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " ..., \n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]]\n",
      "\n",
      "label_0 not_tensor = [[0 0 0 0 0 1 0 0 0 0]\n",
      " [1 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 1 0 0 0 0 0]\n",
      " [0 1 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0 1]\n",
      " [0 0 1 0 0 0 0 0 0 0]\n",
      " [0 1 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 1 0 0 0 0 0 0]\n",
      " [0 1 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 1 0 0 0 0 0]]\n",
      "\n",
      "step_0 loss:  tensor.shape = [], tensor = 35.361572265625\n",
      "\n",
      "training end. step_9 final loss:  tensor.shape = [], tensor = 0.0\n",
      "\n",
      "save_path is:  /tmp/ML_Softmax_Regression_Mnist.vari\n",
      "At evaluate, the _W tensor.shape = [784  10], tensor = [[ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " ..., \n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]]\n",
      "\n",
      "At evaluate, the _b tensor.shape = [10], tensor = [ -9.31322519e-11   9.99999815e-04  -1.30385161e-10  -1.30385161e-10\n",
      "   9.99999815e-04  -9.31322519e-11  -1.00000005e-03  -1.00000005e-03\n",
      "  -1.00000005e-03   9.99999931e-04]\n",
      "\n",
      "At evaluate, test_features not_tensor = [[ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " ..., \n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]]\n",
      "\n",
      "At evaluate, test_label not_tensor = [[0 0 0 0 0 1 0 0 0 0]\n",
      " [1 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 1 0 0 0 0 0]\n",
      " [0 1 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0 1]\n",
      " [0 0 1 0 0 0 0 0 0 0]\n",
      " [0 1 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 1 0 0 0 0 0 0]\n",
      " [0 1 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 1 0 0 0 0 0]]\n",
      "\n",
      "evaluate_result tensor.shape = [], tensor = 0.07999999821186066\n",
      "\n",
      "----------------- do_train: finish -----------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\nfor i in range(2):\\n    features, labels = one_ml.inputs()\\n    \\n    one_ml._echo_tensor( features, '{}_features: '.format(i) )\\n    #one_ml._echo_tensor( labels, '{}_labels: '.format(i) )\\n    print('{}_labels: '.format(i), labels)\\n\\n    combine = tf.matmul(features, _W) + _b\\n    print('combine: ', sess.run(combine) )\\n    \\n    inference = tf.nn.softmax(combine)\\n    print('inference: ', sess.run(inference), 'inference shape: ', inference.get_shape() )\\n\\n    cross_entropy = -tf.reduce_sum(labels * tf.log(inference))\\n    print('cross_entropy: ', sess.run(cross_entropy), 'cross_entropy shape: ', cross_entropy.get_shape() )\\n    \\n    \\n    \\n    \\n    labels_t = tf.convert_to_tensor(labels)\\n    print('labels_t: ', sess.run(labels_t), 'labels_t shape: ', labels_t.get_shape() )\\n    cross = tf.nn.sparse_softmax_cross_entropy_with_logits(inference, labels_t)\\n    print('cross: ', sess.run(cross) )\\n\""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy\n",
    "import tensorflow as tf\n",
    "\n",
    "from common import *\n",
    "from tensorflow.python.platform import gfile\n",
    "\n",
    "def input_mnist_data(file_path):\n",
    "    with gfile.Open(file_path, 'rb') as bytestream:\n",
    "        magic = read_int32(bytestream)\n",
    "        if magic != 2051:\n",
    "            raise ValueError('Invalid magic number %d in MNIST image file: %s' % (magic, file_path))\n",
    "\n",
    "        num_images = read_int32(bytestream)\n",
    "        rows = read_int32(bytestream)\n",
    "        cols = read_int32(bytestream)\n",
    "        buf = bytestream.read( rows * cols * num_images )\n",
    "        data = numpy.frombuffer(buf, dtype = numpy.uint8)\n",
    "        data = data.reshape(num_images, rows * cols * 1)\n",
    "        data = data.astype(numpy.float32)\n",
    "        return data\n",
    "    \n",
    "def input_mnist_label(file_path):    \n",
    "    with gfile.Open(file_path, 'rb') as bytestream:\n",
    "        magic = read_int32( bytestream )\n",
    "        if magic != 2049:\n",
    "            raise ValueError( 'Invalid magic number %d in MNIST label file: %s' % (magic, file_path) )\n",
    "        \n",
    "        num_items = read_int32( bytestream )\n",
    "        buf = bytestream.read(num_items)\n",
    "        labels = numpy.frombuffer(buf, dtype = numpy.uint8)\n",
    "        return dense_to_one_hot( labels, 10 )\n",
    "\n",
    "\n",
    "class ML_Softmax_Regression_Mnist( ML_Softmax_Regression ):\n",
    "    def __init__(self, feature_number, label_number):\n",
    "        ML_Softmax_Regression.__init__(self, feature_number, label_number)\n",
    "        self.__train_data = input_mnist_data('./data_source/train-images.idx3-ubyte')\n",
    "        self.__train_label = input_mnist_label('./data_source/train-labels.idx1-ubyte')\n",
    "        print( '__train_label: ', self.__train_label )\n",
    "        \n",
    "        self.__generator = self.__get_next_data( 10 )\n",
    "        \n",
    "    def __get_next_data(self, batch_size):\n",
    "        step = 0\n",
    "        for step in range( len(self.__train_data) // batch_size ):\n",
    "            begin = step * batch_size\n",
    "            end = begin + batch_size\n",
    "            if end >= len(self.__train_data):\n",
    "                end = len(self.__train_data) - 1\n",
    "    \n",
    "            yield self.__train_data[begin : end], self.__train_label[begin : end]\n",
    "\n",
    "\n",
    "    def inputs(self, file_name = [], batch_size = 10):\n",
    "        return next( self.__generator )\n",
    "\n",
    "    \n",
    "    def loss(self, features, label):\n",
    "        label_predicted = self.combine_inputs(features)\n",
    "        return tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits = label_predicted, labels = label) )\n",
    "            \n",
    "        label_predicted = self.inference(features)\n",
    "        cost = -tf.reduce_sum( label * tf.log( label_predicted ) )\n",
    "        \n",
    "        print( 'in loss _W: ', self._W )\n",
    "        print( 'in loss _b: ', self._b )\n",
    "        print( 'in loss return cost: ', cost )\n",
    "        return cost\n",
    "    \n",
    "\n",
    "one_ml = ML_Softmax_Regression_Mnist(784, 10)\n",
    "one_ml.do_train( 10 )\n",
    "\n",
    "'''\n",
    "_W = tf.Variable(tf.zeros([784, 10]), name = 'weights')\n",
    "_b = tf.Variable(tf.zeros([10]), name = 'bias')\n",
    "sess = tf.Session()\n",
    "sess.run( tf.initialize_all_variables() )\n",
    "'''\n",
    "\n",
    "'''\n",
    "for i in range(2):\n",
    "    features, labels = one_ml.inputs()\n",
    "    \n",
    "    one_ml._echo_tensor( features, '{}_features: '.format(i) )\n",
    "    #one_ml._echo_tensor( labels, '{}_labels: '.format(i) )\n",
    "    print('{}_labels: '.format(i), labels)\n",
    "\n",
    "    combine = tf.matmul(features, _W) + _b\n",
    "    print('combine: ', sess.run(combine) )\n",
    "    \n",
    "    inference = tf.nn.softmax(combine)\n",
    "    print('inference: ', sess.run(inference), 'inference shape: ', inference.get_shape() )\n",
    "\n",
    "    cross_entropy = -tf.reduce_sum(labels * tf.log(inference))\n",
    "    print('cross_entropy: ', sess.run(cross_entropy), 'cross_entropy shape: ', cross_entropy.get_shape() )\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    labels_t = tf.convert_to_tensor(labels)\n",
    "    print('labels_t: ', sess.run(labels_t), 'labels_t shape: ', labels_t.get_shape() )\n",
    "    cross = tf.nn.sparse_softmax_cross_entropy_with_logits(inference, labels_t)\n",
    "    print('cross: ', sess.run(cross) )\n",
    "'''    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}