{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# import the necessary packages\n",
    "from tensorflow.keras.datasets import mnist\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "matplotlib.use(\"Agg\")\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.optimizers import SGD\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from imutils import build_montages\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import cv2\n",
    "from tensorflow.keras.layers import BatchNormalization, Conv2D, AveragePooling2D, MaxPooling2D, ZeroPadding2D, Activation, Dense\n",
    "from tensorflow.keras.layers import Flatten, Input, add\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from tensorflow.keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasetPath = \"az_data.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_az_dataset(datasetPath):\n",
    "\t# initialize the list of data and labels\n",
    "\tdata = []\n",
    "\tlabels = []\n",
    "\t# loop over the rows of the A-Z handwritten digit dataset\n",
    "\tfor row in open(datasetPath):\n",
    "\t\t# parse the label and image from the row\n",
    "\t\trow = row.split(\",\")\n",
    "\t\tlabel = int(row[0])\n",
    "\t\timage = np.array([int(x) for x in row[1:]], dtype=\"uint8\")\n",
    "\t\t# images are represented as single channel (grayscale) images\n",
    "\t\t# that are 28x28=784 pixels -- we need to take this flattened\n",
    "\t\t# 784-d list of numbers and repshape them into a 28x28 matrix\n",
    "\t\timage = image.reshape((28, 28))\n",
    "\t\tdata.append(image)\n",
    "\t\tlabels.append(label)\n",
    "    # convert the data and labels to NumPy arrays\n",
    "\tdata = np.array(data, dtype=\"float32\")\n",
    "\tlabels = np.array(labels, dtype=\"int\")\n",
    "\t# return a 2-tuple of the A-Z data and labels\n",
    "\treturn (data, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist_dataset():\n",
    "\t# load the MNIST dataset and stack the training data and testing\n",
    "\t# data together (we'll create our own training and testing splits\n",
    "\t# later in the project)\n",
    "\t((trainData, trainLabels), (testData, testLabels)) = mnist.load_data()\n",
    "\tdata = np.vstack([trainData, testData])\n",
    "\tlabels = np.hstack([trainLabels, testLabels])\n",
    "\t# return a 2-tuple of the MNIST data and labels\n",
    "\treturn (data, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# construct the argument parser and parse the arguments\n",
    "# ap = argparse.ArgumentParser()\n",
    "# ap.add_argument(\"-a\", \"--az\", required=True,\n",
    "# \thelp=\"path to A-Z dataset\")th\n",
    "# ap.add_argument(\"-m\", \"--model\", type=str, required=True,\n",
    "# \thelp=\"path to output trained handwriting recognition model\")\n",
    "# ap.add_argument(\"-p\", \"--plot\", type=str, default=\"plot.png\",\n",
    "# \thelp=\"path to output training history file\")\n",
    "# args = vars(ap.parse_args())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def residual_module(data, K, stride, chanDim, red=False,reg=0.0001,bnEps=2e-5, bnMom=0.9):\n",
    "    shortcut = data\n",
    "    print(data)\n",
    "    \n",
    "    bn1 = BatchNormalization(axis=chanDim, epsilon=bnEps, momentum=bnMom)(data)\n",
    "    act1 = Activation(\"relu\")(bn1)\n",
    "    conv1 = Conv2D(int(K * 0.25), (1,1), use_bias=False, kernel_regularizer=l2(reg))(act1)\n",
    "        \n",
    "    bn2 = BatchNormalization(axis=chanDim, epsilon=bnEps, momentum=bnMom)(conv1)\n",
    "    act2 = Activation(\"relu\")(bn2)\n",
    "    conv2 = Conv2D(int(K * 0.25), (3,3), strides=stride, padding=\"same\", use_bias=False, \n",
    "                kernel_regularizer=l2(reg))(act2)\n",
    "\n",
    "    bn3 = BatchNormalization(axis=chanDim, epsilon=bnEps, momentum=bnMom)(conv2)\n",
    "    act3 = Activation(\"relu\")(bn3)\n",
    "    conv3 = Conv2D(K, (1,1), use_bias=False, kernel_regularizer=l2(reg))(act3)\n",
    "\n",
    "    if red:\n",
    "        shortcut = Conv2D(K, (1,1), strides=stride, use_bias=False,kernel_regularizer=l2(reg))(act1)\n",
    "\n",
    "    x = add([conv3, shortcut])\n",
    "\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build(width, height, depth, classes, stages, filters, reg=0.0001, bnEps=2e-5, bnMom=0.9):\n",
    "    inputShape = (height, width, depth)\n",
    "    chanDim = -1\n",
    "   \n",
    "    if K.image_data_format() == \"channels_first\":\n",
    "        inputShape = (depth, height, width)\n",
    "        chanDim = 1\n",
    "\n",
    "    inputs = Input(shape=inputShape)\n",
    "    x = BatchNormalization(axis=chanDim, epsilon=bnEps, momentum=bnMom)(inputs)\n",
    "\n",
    "    x = Conv2D(filters[0], (5,5), use_bias=False, padding=\"same\", kernel_regularizer=l2(reg))(x)\n",
    "    x = BatchNormalization(axis=chanDim, epsilon=bnEps, momentum=bnMom)(x)\n",
    "    x = Activation(\"relu\")(x)\n",
    "    x = ZeroPadding2D((1, 1))(x)\n",
    "    x = MaxPooling2D((3,3), strides=(2, 2))(x)\n",
    "\n",
    "    for i in range(0, len(stages)):\n",
    "        stride = (1, 1) if i == 0 else (2,2)\n",
    "        x = residual_module(x, filters[i + 1], stride, chanDim, red=True, bnEps=bnEps, bnMom=bnMom)\n",
    "     \n",
    "        for j in range(0, stages[i] - 1):\n",
    "            x = residual_module(x, filters[i + 1], (1, 1), chanDim, bnEps=bnEps, bnMom=bnMom)\n",
    "\n",
    "    x = BatchNormalization(axis=chanDim, epsilon=bnEps, momentum=bnMom)(x)\n",
    "    x = Activation(\"relu\")(x)\n",
    "    x = MaxPooling2D((2,2), padding='same')(x)\n",
    "  #  x = AveragePooling2D((8,8))(x)\n",
    "\n",
    "    x = Flatten()(x)\n",
    "    x = Dense(classes, kernel_regularizer=l2(reg))(x)\n",
    "    x = Activation(\"softmax\")(x)\n",
    "\n",
    "    model = Model(inputs, x, name=\"resnet\")\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] loading datasets...\n"
     ]
    }
   ],
   "source": [
    "# load the A-Z and MNIST datasets, respectively\n",
    "print(\"[INFO] loading datasets...\")\n",
    "(azData, azLabels) = load_az_dataset(datasetPath)\n",
    "(digitsData, digitsLabels) = load_mnist_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "(digitsData, digitsLabels) = load_mnist_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, ..., 4, 5, 6], dtype=uint8)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "digitsLabels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "azLabels[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the MNIST dataset occupies the labels 0-9, so let's add 10 to every\n",
    "# A-Z label to ensure the A-Z characters are not incorrectly labeled\n",
    "# as digits\n",
    "azLabels += 10\n",
    "# i =0\n",
    "# for label in azLabels:\n",
    "#     azLabels[i] = label + 10\n",
    "#     i += 1\n",
    "# stack the A-Z data and labels with the MNIST digits data and labels\n",
    "data = np.vstack([azData, digitsData])\n",
    "labels = np.hstack([azLabels, digitsLabels])\n",
    "# each image in the A-Z and MNIST digts datasets are 28x28 pixels;\n",
    "# however, the architecture we're using is designed for 32x32 images,\n",
    "# so we need to resize them to 32x32\n",
    "data = [cv2.resize(image, (32, 32)) for image in data]\n",
    "data = np.array(data, dtype=\"float32\")\n",
    "# add a channel dimension to every image in the dataset and scale the\n",
    "# pixel intensities of the images from [0, 255] down to [0, 1]\n",
    "data = np.expand_dims(data, axis=-1)\n",
    "data /= 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert the labels from integers to vectors\n",
    "le = LabelBinarizer()\n",
    "labels = le.fit_transform(labels)\n",
    "counts = labels.sum(axis=0)\n",
    "# account for skew in the labeled data\n",
    "classTotals = labels.sum(axis=0)\n",
    "classWeight = {}\n",
    "# loop over all classes and calculate the class weight\n",
    "for i in range(0, len(classTotals)):\n",
    "\tclassWeight[i] = classTotals.max() / classTotals[i]\n",
    "# partition the data into training and testing splits using 80% of\n",
    "# the data for training and the remaining 20% for testing\n",
    "(trainX, testX, trainY, testY) = train_test_split(data,\n",
    "\tlabels, test_size=0.20, stratify=labels, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize the number of epochs to train for, initial learning rate,\n",
    "# and batch size\n",
    "EPOCHS = 10\n",
    "INIT_LR = 1e-1\n",
    "BS = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "aug = ImageDataGenerator(\n",
    "\trotation_range=10,\n",
    "\tzoom_range=0.05,\n",
    "\twidth_shift_range=0.1,\n",
    "\theight_shift_range=0.1,\n",
    "\tshear_range=0.15,\n",
    "\thorizontal_flip=False,\n",
    "\tfill_mode=\"nearest\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] compiling model...\n",
      "Tensor(\"max_pooling2d_9/Identity:0\", shape=(None, 16, 16, 64), dtype=float32)\n",
      "Tensor(\"add_24/Identity:0\", shape=(None, 16, 16, 64), dtype=float32)\n",
      "Tensor(\"add_25/Identity:0\", shape=(None, 16, 16, 64), dtype=float32)\n",
      "Tensor(\"add_26/Identity:0\", shape=(None, 16, 16, 64), dtype=float32)\n",
      "Tensor(\"add_27/Identity:0\", shape=(None, 8, 8, 128), dtype=float32)\n",
      "Tensor(\"add_28/Identity:0\", shape=(None, 8, 8, 128), dtype=float32)\n",
      "Tensor(\"add_29/Identity:0\", shape=(None, 8, 8, 128), dtype=float32)\n",
      "Tensor(\"add_30/Identity:0\", shape=(None, 4, 4, 256), dtype=float32)\n",
      "Tensor(\"add_31/Identity:0\", shape=(None, 4, 4, 256), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# initialize and compile our deep neural network\n",
    "print(\"[INFO] compiling model...\")\n",
    "opt = SGD(lr=INIT_LR, decay=INIT_LR / EPOCHS)\n",
    "model = build(32, 32, 1, len(le.classes_), (3, 3, 3),(64, 64, 128, 256), reg=0.0005)\n",
    "model.compile(loss=\"categorical_crossentropy\", optimizer=opt,metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] training network...\n",
      "Epoch 1/10\n",
      "2765/2765 [==============================] - 4128s 1s/step - loss: 2.8696 - accuracy: 0.8201 - val_loss: 0.6788 - val_accuracy: 0.9390\n",
      "Epoch 2/10\n",
      "2765/2765 [==============================] - 4080s 1s/step - loss: 1.2489 - accuracy: 0.9154 - val_loss: 0.6867 - val_accuracy: 0.9116\n",
      "Epoch 3/10\n",
      "2765/2765 [==============================] - 4041s 1s/step - loss: 1.1348 - accuracy: 0.9234 - val_loss: 0.6944 - val_accuracy: 0.9003\n",
      "Epoch 4/10\n",
      "2765/2765 [==============================] - 4918s 2s/step - loss: 1.0831 - accuracy: 0.9275 - val_loss: 0.6536 - val_accuracy: 0.9209\n",
      "Epoch 5/10\n",
      "2765/2765 [==============================] - 5328s 2s/step - loss: 1.0486 - accuracy: 0.9300 - val_loss: 0.6986 - val_accuracy: 0.8915\n",
      "Epoch 6/10\n",
      "2765/2765 [==============================] - 5326s 2s/step - loss: 1.0234 - accuracy: 0.9318 - val_loss: 0.6492 - val_accuracy: 0.9162\n",
      "Epoch 7/10\n",
      "2765/2765 [==============================] - 5239s 2s/step - loss: 1.0073 - accuracy: 0.9328 - val_loss: 0.6762 - val_accuracy: 0.9003\n",
      "Epoch 8/10\n",
      "2765/2765 [==============================] - 5038s 2s/step - loss: 0.9892 - accuracy: 0.9353 - val_loss: 0.6551 - val_accuracy: 0.9114\n",
      "Epoch 9/10\n",
      "2765/2765 [==============================] - 4271s 2s/step - loss: 0.9754 - accuracy: 0.9356 - val_loss: 0.6494 - val_accuracy: 0.9117\n",
      "Epoch 10/10\n",
      "2765/2765 [==============================] - 4089s 1s/step - loss: 0.9657 - accuracy: 0.9362 - val_loss: 0.6802 - val_accuracy: 0.8951\n",
      "[INFO] evaluating network...\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.15      0.91      0.26      1381\n",
      "           1       0.97      0.99      0.98      1575\n",
      "           2       0.93      0.95      0.94      1398\n",
      "           3       0.98      0.99      0.98      1428\n",
      "           4       0.91      0.97      0.94      1365\n",
      "           5       0.71      0.91      0.80      1263\n",
      "           6       0.95      0.96      0.96      1375\n",
      "           7       0.97      0.99      0.98      1459\n",
      "           8       0.95      0.98      0.96      1365\n",
      "           9       0.98      0.99      0.98      1392\n",
      "           A       0.99      0.99      0.99      2774\n",
      "           B       0.98      0.97      0.98      1734\n",
      "           C       0.99      0.98      0.99      4682\n",
      "           D       0.92      0.97      0.94      2027\n",
      "           E       0.99      0.98      0.99      2288\n",
      "           F       0.97      0.99      0.98       232\n",
      "           G       0.95      0.95      0.95      1152\n",
      "           H       0.97      0.98      0.97      1444\n",
      "           I       0.98      0.99      0.98       224\n",
      "           J       0.96      0.97      0.97      1699\n",
      "           K       0.97      0.98      0.98      1121\n",
      "           L       0.97      0.99      0.98      2317\n",
      "           M       1.00      0.99      0.99      2467\n",
      "           N       0.99      0.99      0.99      3802\n",
      "           O       0.98      0.37      0.54     11565\n",
      "           P       1.00      0.99      0.99      3868\n",
      "           Q       0.97      0.98      0.98      1162\n",
      "           R       0.99      0.99      0.99      2313\n",
      "           S       0.99      0.95      0.97      9684\n",
      "           T       0.99      0.99      0.99      4499\n",
      "           U       0.99      0.98      0.98      5802\n",
      "           V       0.98      1.00      0.99       836\n",
      "           W       0.99      0.98      0.99      2157\n",
      "           X       0.98      0.99      0.99      1254\n",
      "           Y       0.98      0.95      0.97      2172\n",
      "           Z       0.95      0.93      0.94      1215\n",
      "\n",
      "    accuracy                           0.90     88491\n",
      "   macro avg       0.94      0.96      0.94     88491\n",
      "weighted avg       0.96      0.90      0.91     88491\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# train the network\n",
    "print(\"[INFO] training network...\")\n",
    "H = model.fit(\n",
    "\taug.flow(trainX, trainY, batch_size=BS),\n",
    "\tvalidation_data=(testX, testY),\n",
    "\tsteps_per_epoch=len(trainX) // BS,\n",
    "\tepochs=EPOCHS,\n",
    "\tclass_weight=classWeight,\n",
    "\tverbose=1)\n",
    "# define the list of label names\n",
    "labelNames = \"0123456789\"\n",
    "labelNames += \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n",
    "labelNames = [l for l in labelNames]\n",
    "# evaluate the network\n",
    "print(\"[INFO] evaluating network...\")\n",
    "predictions = model.predict(testX, batch_size=BS)\n",
    "print(classification_report(testY.argmax(axis=1),\n",
    "\tpredictions.argmax(axis=1), target_names=labelNames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] serializing network...\n"
     ]
    }
   ],
   "source": [
    "# save the model to disk\n",
    "print(\"[INFO] serializing network...\")\n",
    "model.save(\"handwriting_recognition.h5\", save_format=\"h5\")\n",
    "# construct a plot that plots and saves the training history\n",
    "N = np.arange(0, EPOCHS)\n",
    "plt.style.use(\"ggplot\")\n",
    "plt.figure()\n",
    "plt.plot(N, H.history[\"loss\"], label=\"train_loss\")\n",
    "plt.plot(N, H.history[\"val_loss\"], label=\"val_loss\")\n",
    "plt.title(\"Training Loss and Accuracy\")\n",
    "plt.xlabel(\"Epoch #\")\n",
    "plt.ylabel(\"Loss/Accuracy\")\n",
    "plt.legend(loc=\"lower left\")\n",
    "plt.savefig(\"graph.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# initialize our list of output test images\n",
    "images = []\n",
    "# randomly select a few testing characters\n",
    "for i in np.random.choice(np.arange(0, len(testY)), size=(49,)):\n",
    "\t# classify the character\n",
    "\tprobs = model.predict(testX[np.newaxis, i])\n",
    "\tprediction = probs.argmax(axis=1)\n",
    "\tlabel = labelNames[prediction[0]]\n",
    "\t# extract the image from the test data and initialize the text\n",
    "\t# label color as green (correct)\n",
    "\timage = (testX[i] * 255).astype(\"uint8\")\n",
    "\tcolor = (0, 255, 0)\n",
    "\t# otherwise, the class label prediction is incorrect\n",
    "\tif prediction[0] != np.argmax(testY[i]):\n",
    "\t\tcolor = (0, 0, 255)\n",
    "\t# merge the channels into one image, resize the image from 32x32\n",
    "\t# to 96x96 so we can better see it and then draw the predicted\n",
    "\t# label on the image\n",
    "\timage = cv2.merge([image] * 3)\n",
    "\timage = cv2.resize(image, (96, 96), interpolation=cv2.INTER_LINEAR)\n",
    "\tcv2.putText(image, label, (5, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75,\n",
    "\t\tcolor, 2)\n",
    "\t# add the image to our list of output images\n",
    "\timages.append(image)\n",
    "# construct the montage for the images\n",
    "montage = build_montages(images, (96, 96), (7, 7))[0]\n",
    "# show the output montage\n",
    "cv2.imshow(\"OCR Results\", montage)\n",
    "cv2.waitKey(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
