Abstractive_Text_Summarization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Abstractive_Text_Summarization.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/IsinghGitHub/CellStart/blob/master/Abstractive_Text_Summarization.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AON6TxqtAANP",
        "colab_type": "code",
        "outputId": "c0f9ba0c-395f-428b-9fe4-4f12dc78244b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive/')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D9d5ShFeDEsw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cd '/content/gdrive/My Drive/CellStrat/Text_Summarization/Abstractive'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qhuq4AbaA_lI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!python3 '/content/gdrive/My Drive/CellStrat/Text_Summarization/Abstractive/attention.py'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iDJGoJgsFath",
        "colab_type": "code",
        "outputId": "9aa76c4d-667b-4f7d-8ac5-f9235df51d10",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "source": [
        "# Upload files\n",
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-dcf841e4-52e3-4cf8-b355-1d036a7ad079\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-dcf841e4-52e3-4cf8-b355-1d036a7ad079\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving attention.py to attention.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hJeGEEsWBFxP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import attention"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zwq-J3Y9EBI0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from attention import AttentionLayer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZeiFAUtwGUpE",
        "colab_type": "text"
      },
      "source": [
        "## Importing the required libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pk1o8uAMGN9z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ccca2cf7-4850-4659-fbd5-ff01ecb5efcd"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd \n",
        "import re\n",
        "from bs4 import BeautifulSoup\n",
        "from keras.preprocessing.text import Tokenizer \n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from nltk.corpus import stopwords\n",
        "from tensorflow.keras.layers import Input, LSTM, Embedding, Dense, Concatenate, TimeDistributed\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "import warnings\n",
        "pd.set_option(\"display.max_colwidth\", 200)\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Girel9TJGl9N",
        "colab_type": "text"
      },
      "source": [
        "# Read the dataset\n",
        "We’ll take a sample of 100,000 reviews to reduce the training time of our model. Feel free to use the entire dataset for training your model if your machine has that kind of computational power."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OPCDw7jAGcyB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data=pd.read_csv(\"/content/gdrive/My Drive/CellStrat/Text_Summarization/Abstractive/amazon-fine-food-reviews/Reviews.csv\",nrows=100000)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z-sv6f2sHBEp",
        "colab_type": "code",
        "outputId": "d25efe89-fb38-4d14-8ab9-a2bad7d360c4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        }
      },
      "source": [
        "data.head(5)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Id</th>\n",
              "      <th>ProductId</th>\n",
              "      <th>UserId</th>\n",
              "      <th>ProfileName</th>\n",
              "      <th>HelpfulnessNumerator</th>\n",
              "      <th>HelpfulnessDenominator</th>\n",
              "      <th>Score</th>\n",
              "      <th>Time</th>\n",
              "      <th>Summary</th>\n",
              "      <th>Text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>B001E4KFG0</td>\n",
              "      <td>A3SGXH7AUHU8GW</td>\n",
              "      <td>delmartian</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>1303862400</td>\n",
              "      <td>Good Quality Dog Food</td>\n",
              "      <td>I have bought several of the Vitality canned dog food products and have found them all to be of good quality. The product looks more like a stew than a processed meat and it smells better. My Labr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>B00813GRG4</td>\n",
              "      <td>A1D87F6ZCVE5NK</td>\n",
              "      <td>dll pa</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1346976000</td>\n",
              "      <td>Not as Advertised</td>\n",
              "      <td>Product arrived labeled as Jumbo Salted Peanuts...the peanuts were actually small sized unsalted. Not sure if this was an error or if the vendor intended to represent the product as \"Jumbo\".</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>B000LQOCH0</td>\n",
              "      <td>ABXLMWJIXXAIN</td>\n",
              "      <td>Natalia Corres \"Natalia Corres\"</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>1219017600</td>\n",
              "      <td>\"Delight\" says it all</td>\n",
              "      <td>This is a confection that has been around a few centuries.  It is a light, pillowy citrus gelatin with nuts - in this case Filberts. And it is cut into tiny squares and then liberally coated with ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>B000UA0QIQ</td>\n",
              "      <td>A395BORC6FGVXV</td>\n",
              "      <td>Karl</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>1307923200</td>\n",
              "      <td>Cough Medicine</td>\n",
              "      <td>If you are looking for the secret ingredient in Robitussin I believe I have found it.  I got this in addition to the Root Beer Extract I ordered (which was good) and made some cherry soda.  The fl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>B006K2ZZ7K</td>\n",
              "      <td>A1UQRSCLF8GW1T</td>\n",
              "      <td>Michael D. Bigham \"M. Wassir\"</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>1350777600</td>\n",
              "      <td>Great taffy</td>\n",
              "      <td>Great taffy at a great price.  There was a wide assortment of yummy taffy.  Delivery was very quick.  If your a taffy lover, this is a deal.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Id  ...                                                                                                                                                                                                     Text\n",
              "0   1  ...  I have bought several of the Vitality canned dog food products and have found them all to be of good quality. The product looks more like a stew than a processed meat and it smells better. My Labr...\n",
              "1   2  ...           Product arrived labeled as Jumbo Salted Peanuts...the peanuts were actually small sized unsalted. Not sure if this was an error or if the vendor intended to represent the product as \"Jumbo\".\n",
              "2   3  ...  This is a confection that has been around a few centuries.  It is a light, pillowy citrus gelatin with nuts - in this case Filberts. And it is cut into tiny squares and then liberally coated with ...\n",
              "3   4  ...  If you are looking for the secret ingredient in Robitussin I believe I have found it.  I got this in addition to the Root Beer Extract I ordered (which was good) and made some cherry soda.  The fl...\n",
              "4   5  ...                                                             Great taffy at a great price.  There was a wide assortment of yummy taffy.  Delivery was very quick.  If your a taffy lover, this is a deal.\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OtfTMNyeHPnh",
        "colab_type": "text"
      },
      "source": [
        "## Drop Duplicates and NA values"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mCoAL6T7HFhB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data.drop_duplicates(subset=['Text'],inplace=True)  #dropping the duplicates\n",
        "data.dropna(axis =0,inplace=True) #dropping na"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B-VOH59yJWdE",
        "colab_type": "text"
      },
      "source": [
        "## Let us look at datatypes and shape of the dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zaiGWXWoHnVS",
        "colab_type": "code",
        "outputId": "8eef88cf-3afe-4e05-cbfc-646cc525f4d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        }
      },
      "source": [
        "data.info()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 88421 entries, 0 to 99999\n",
            "Data columns (total 10 columns):\n",
            "Id                        88421 non-null int64\n",
            "ProductId                 88421 non-null object\n",
            "UserId                    88421 non-null object\n",
            "ProfileName               88421 non-null object\n",
            "HelpfulnessNumerator      88421 non-null int64\n",
            "HelpfulnessDenominator    88421 non-null int64\n",
            "Score                     88421 non-null int64\n",
            "Time                      88421 non-null int64\n",
            "Summary                   88421 non-null object\n",
            "Text                      88421 non-null object\n",
            "dtypes: int64(5), object(5)\n",
            "memory usage: 7.4+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ITpEo-FbJrB_",
        "colab_type": "text"
      },
      "source": [
        "## Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TX2V6WWJJZWs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\",\n",
        "                           \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\",\n",
        "                           \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",\n",
        "                           \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\",\n",
        "                           \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n",
        "                           \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\",\n",
        "                           \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\",\n",
        "                           \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\",\n",
        "                           \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\",\n",
        "                           \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\",\n",
        "                           \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\",\n",
        "                           \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\",\n",
        "                           \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\",\n",
        "                           \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\",\n",
        "                           \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\",\n",
        "                           \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",\n",
        "                           \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\",\n",
        "                           \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\",\n",
        "                           \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\",\n",
        "                           \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\",\n",
        "                           \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\n",
        "                           \"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\",\n",
        "                           \"you're\": \"you are\", \"you've\": \"you have\"}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8jAt1YfCMNNF",
        "colab_type": "text"
      },
      "source": [
        "### We will perform the below preprocessing tasks for our data:\n",
        "\n",
        "1.Convert everything to lowercase\n",
        "\n",
        "2.Remove HTML tags\n",
        "\n",
        "3.Contraction mapping\n",
        "\n",
        "4.Remove (‘s)\n",
        "\n",
        "5.Remove any text inside the parenthesis ( )\n",
        "\n",
        "6.Eliminate punctuations and special characters\n",
        "\n",
        "7.Remove stopwords\n",
        "\n",
        "8.Remove short words\n",
        "\n",
        "Let’s define the function:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mfG9qrjpUIhT",
        "colab_type": "code",
        "outputId": "081b7090-55bf-4817-bd23-c27cfa6e9507",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SvifSmsZMHmR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "stop_words = set(stopwords.words('english')) \n",
        "\n",
        "def text_cleaner(text,num):\n",
        "    newString = text.lower()\n",
        "    newString = BeautifulSoup(newString, \"lxml\").text\n",
        "    newString = re.sub(r'\\([^)]*\\)', '', newString)\n",
        "    newString = re.sub('\"','', newString)\n",
        "    newString = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in newString.split(\" \")])    \n",
        "    newString = re.sub(r\"'s\\b\",\"\",newString)\n",
        "    newString = re.sub(\"[^a-zA-Z]\", \" \", newString) \n",
        "    newString = re.sub('[m]{2,}', 'mm', newString)\n",
        "    if(num==0):\n",
        "        tokens = [w for w in newString.split() if not w in stop_words]\n",
        "    else:\n",
        "        tokens=newString.split()\n",
        "    long_words=[]\n",
        "    for i in tokens:\n",
        "        if len(i)>1:                                                 #removing short word\n",
        "            long_words.append(i)   \n",
        "    return (\" \".join(long_words)).strip()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kSQIrLboTtxQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#call the function\n",
        "cleaned_text = []\n",
        "for t in data['Text']:\n",
        "    cleaned_text.append(text_cleaner(t,0))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SCyzQ6LKUTl6",
        "colab_type": "code",
        "outputId": "cb0a9adb-5f82-4c7f-e36f-b6b9c52a16c6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "cleaned_text[:5]"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['bought several vitality canned dog food products found good quality product looks like stew processed meat smells better labrador finicky appreciates product better',\n",
              " 'product arrived labeled jumbo salted peanuts peanuts actually small sized unsalted sure error vendor intended represent product jumbo',\n",
              " 'confection around centuries light pillowy citrus gelatin nuts case filberts cut tiny squares liberally coated powdered sugar tiny mouthful heaven chewy flavorful highly recommend yummy treat familiar story lewis lion witch wardrobe treat seduces edmund selling brother sisters witch',\n",
              " 'looking secret ingredient robitussin believe found got addition root beer extract ordered made cherry soda flavor medicinal',\n",
              " 'great taffy great price wide assortment yummy taffy delivery quick taffy lover deal']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hGIMeJ2tWBuh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#call the function\n",
        "cleaned_summary = []\n",
        "for t in data['Summary']:\n",
        "    cleaned_summary.append(text_cleaner(t,1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G3OksaqLXqkl",
        "colab_type": "code",
        "outputId": "1a0a91a1-5430-4f87-834b-60f8b1fcc828",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "cleaned_summary[:10]"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['good quality dog food',\n",
              " 'not as advertised',\n",
              " 'delight says it all',\n",
              " 'cough medicine',\n",
              " 'great taffy',\n",
              " 'nice taffy',\n",
              " 'great just as good as the expensive brands',\n",
              " 'wonderful tasty taffy',\n",
              " 'yay barley',\n",
              " 'healthy dog food']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dIFYm-uWX0Yh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data['cleaned_text']=cleaned_text\n",
        "data['cleaned_summary']=cleaned_summary"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D3yRQOjcYphG",
        "colab_type": "code",
        "outputId": "dccdbdc4-6ad1-4c70-a40d-4cb473dea062",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199
        }
      },
      "source": [
        "data.head(1)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Id</th>\n",
              "      <th>ProductId</th>\n",
              "      <th>UserId</th>\n",
              "      <th>ProfileName</th>\n",
              "      <th>HelpfulnessNumerator</th>\n",
              "      <th>HelpfulnessDenominator</th>\n",
              "      <th>Score</th>\n",
              "      <th>Time</th>\n",
              "      <th>Summary</th>\n",
              "      <th>Text</th>\n",
              "      <th>cleaned_text</th>\n",
              "      <th>cleaned_summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>B001E4KFG0</td>\n",
              "      <td>A3SGXH7AUHU8GW</td>\n",
              "      <td>delmartian</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>1303862400</td>\n",
              "      <td>Good Quality Dog Food</td>\n",
              "      <td>I have bought several of the Vitality canned dog food products and have found them all to be of good quality. The product looks more like a stew than a processed meat and it smells better. My Labr...</td>\n",
              "      <td>bought several vitality canned dog food products found good quality product looks like stew processed meat smells better labrador finicky appreciates product better</td>\n",
              "      <td>good quality dog food</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Id  ...        cleaned_summary\n",
              "0   1  ...  good quality dog food\n",
              "\n",
              "[1 rows x 12 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lQ4hBEXkY3XN",
        "colab_type": "text"
      },
      "source": [
        "## Drop empty rows"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EGB1QdgzYr8D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data.replace('', np.nan, inplace=True)\n",
        "data.dropna(axis=0,inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "26ePFPkgY_Ju",
        "colab_type": "text"
      },
      "source": [
        "## Understanding the distribution of the sequences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ljt-_UpDY7j-",
        "colab_type": "code",
        "outputId": "77998d78-d355-48d1-cbb7-df047a80c965",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "text_word_count=[]\n",
        "summary_word_count=[]\n",
        "\n",
        "#Populate the list with sentence Lengths\n",
        "\n",
        "for i in data['cleaned_text']:\n",
        "  text_word_count.append(len(i.split()))\n",
        "for i in data['cleaned_summary']:\n",
        "    summary_word_count.append(len(i.split()))\n",
        "\n",
        "length_df = pd.DataFrame({'text':text_word_count, 'summary':summary_word_count})\n",
        "\n",
        "length_df.hist(bins = 30)\n",
        "plt.show()\n",
        "  "
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEICAYAAAC0+DhzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+01XWd7/HnK03zWgZonRBssIma\nZTKRcJVZdZtTJiI1YXeVQd5AY0kttbF1WRU2rUWjOZfupI3OeC1KrtCY6NVMpjA6kXuZ6w4KJImg\nDkfC62EhFKB0qCzoff/4fnZ82d999tlwfuwfvB5r7bX3fn8/3+/+fs767vPe38/38/18FBGYmZnl\nvaLRO2BmZs3HycHMzAqcHMzMrMDJwczMCpwczMyswMnBzMwKnBzMzKzAycHMmpqkbZLeNwjbuUPS\nlwdjn44FTg5WN0nHN3ofzGx4ODkMM0mfl7Rd0q8lPSPp/MpfNJI6JfXk3m+T9FlJT0jaL+l2SR2S\nHkzb+bGkkansOEkh6XJJz0vaK+lTkv5zWv9FSf+S2/afS/qJpN2SfiXpTkkjKj7785KeAPan/biv\nok63SLp5SP9wdkyS9G3gjcC/SeqV9DlJUyT933Qs/1xSZyo7SlKPpL9J718tqVvSbEnzgEuBz6Xt\n/FvDKtUqIsKPYXoAbwWeB05P78cBfw7cAXw5V64T6Mm93wasATqAMcAu4GfAO4BXAT8BFua2GcDX\n07KpwO+A7wGvz63/16n8m4ELgBOB1wEPA/9U8dkbgDOAk4DRwH5gRFp+fNrepEb/ff1oz0c6Bt+X\nXo8BdgPTyX7cXpDevy4tnwq8kI71bwL35rZz2PfMj9oPnzkMr4Nk/4TPkvTKiNgWEc/Wue4/R8TO\niNgO/BR4NCIej4jfAfeTJYq86yPidxHxI7J/5ndFxK7c+u8AiIjuiOiKiJcj4pfATcBfV2zrloh4\nPiJ+GxE7yBLIR9KyacCvImL9Ef0lzI7OfwNWRsTKiPhjRHQB68iSBel4/z/A6hT7ZMP2tMU5OQyj\niOgGPgN8Cdglabmk0+tcfWfu9W+rvH/10ZRPzVPLU1PXPuBfgdMqtvV8xfulZF9S0vO366yD2UD9\nGfCR1KT0oqQXgXeRndGWLQbOBu6IiN2N2Ml24OQwzCLiOxHxLrKDPICvkP2y/0+5Ym8Yxl36h7Qf\nEyLiFLJ/9qooUzl07/eAv5R0NvAB4M4h30s7luWPv+eBb0fEiNzj5IhYBCDpOLLksAy4UtKb+9iO\n9cPJYRhJequk90o6kew6wG+BP5K16U9PF9TeQHZ2MVxeA/QCL0kaA3y2vxVSU9a9wHeAxyLi/w3t\nLtoxbifwpvT6X4G/kXShpOMkvSp14Bibln+BLAl8AvhHYFlKGJXbsX44OQyvE4FFwK84dNHsWrJm\nmZ+TXXj7EXD3MO7T3wPnAC8BPwC+W+d6S4EJuEnJht7/AL6YmpA+CswgSwK/JDuT+CzwCkmTgP8O\nzI6Ig2Rn5QEsSNu5nex634uSvjfMdWg5SlfxzY6IpDcCTwNviIh9jd4fMxtcPnOwIybpFWS/0JY7\nMZi1J9/xakdE0slkbbfPkXVjNbM25GYlMzMr6LdZSdIZkh6StFnSJknXpPgoSV2StqTn8vANSsMp\ndKfhGs7JbWtOKr9F0pxcfJKkjWmdWyRVdqU0M7Nh1O+Zg6TRwOiI+Jmk1wDrgYuBy4A9EbFI0gJg\nZER8XtJ04NNkdyeeB9wcEedJGkV2J+Nksh4E68mGXNgr6THgb4FHgZVkd+Q+WGu/TjvttBg3bhz7\n9+/n5JNPPuo/QDNwHRpj/fr1v4qI1zV6P+pVPuYrteLfvh6u19Co+7g/0vE2gAfIxjN5hixpQHZ3\n4jPp9TeAWbnyz6Tls4Bv5OLfSLHRwNO5+GHl+npMmjQpIiIeeuihaHWuQ2MA66IJxrCp91E+5iu1\n4t++Hq7X0Kj3uD+iC9KSxpGNyfMo0BHZODuQ9dnvSK/HcPhwCz0pViveUyVe7fPnAfMAOjo6KJVK\n9Pb2UiqVjqQaTcd1MLNmU3dykPRq4D7gMxGxL39ZICJC0pBf2Y6IxWS3xjN58uTo7OykVCrR2dk5\n1B89pFwHM2s2dd3nIOmVZInhzogo30G7M12PKF+X2JXi28mGdy4bm2K14mOrxM3MrEHq6a0kstvO\nn4qIm3KLVgDlHkdzyK5FlOOzU6+lKcBLqflpFTBV0sjUs2kqsCot25cm8BAwO7ctMzNrgHqald4J\nfBzYKGlDin2BbIygeyTNJbsh6pK0bCVZT6Vu4DfA5QARsUfS9cDaVO66iNiTXl9JNhHHScCD6WFm\nZg3Sb3KIiEcoDuFcdn6V8gFc1ce2lgBLqsTXkY2/bmZmTcBjK5mZWYGTg5mZFTg5mJlZwTExKuu4\nBT847P22Re9v0J6YDQ0f4zbYfOZgZmYFTg5mZlbg5GBmZgVODmZmVuDkYGZmBU4OZmZW4ORgVoWk\nEZLulfS0pKck/ZWnxrVjiZODWXU3Az+MiL8A3g48BSwAVkfEeGB1eg9wETA+PeYBt0E2zzqwkGy6\n3HOBheWEkspckVtv2jDUyaxuTg5mFSS9Fng32VD1RMTvI+JFYAawNBVbSjaXOim+LM3CuAYYkeY4\nuRDoiog9EbEX6AKmpWWnRMSaNFDlsty2zJrCMXGHtNkROhP4JfC/Jb0dWA9cQ5NMjVupt7eX+RMO\nHhZrhylb23Xq2Vapl5ODWdHxwDnApyPiUUk3c6gJCWjs1LiVSqUSNz6y/7DYtkuL5VpNu0492yr1\ncrOSWVEP0BMRj6b395IlC0+Na8cMJwezChHxAvC8pLem0PnAZjw1rh1D3KxkVt2ngTslnQBsJZvu\n9hV4alw7RvSbHCQtAT4A7IqIs1PsbqD8q2oE8GJETJQ0jqzL3zNp2ZqI+FRaZxKHvgwrgWtSu+0o\n4G5gHLANuCT17DBrmIjYAEyusshT49oxoZ5mpTuo6IMdER+NiIkRMRG4D/hubvGz5WXlxJD01a+7\nr77jZmbWIP0mh4h4GNhTbVlqL70EuKvWNvrp191X33EzM2uQgV5z+C/AzojYkoudKelxYB/wxYj4\nKbX7dffVd7ygWp/vevoMz59w4LD3zdbHuFX6PdfSDnUws0MGmhxmcfhZww7gjRGxO11j+J6kt9W7\nsf76jlfr811Pn+HLKqdQbLI+4K3S77mWdqiDmR1y1MlB0vHAfwUmlWMR8TLwcnq9XtKzwFuo3a97\np6TREbGjou+4mZk1yEDuc3gf8HRE/Km5SNLrJB2XXr+J7MLz1n76dffVd9zMzBqk3+Qg6S7g34G3\nSupJfbwBZlK8EP1u4AlJG8juKv1URb/ub5H1BX+WQ/26FwEXSNpClnAWDaA+ZmY2CPptVoqIWX3E\nL6sSu4+sa2u18lX7dUfEbqr0HTczs8bx8BlmZlbg5GBmZgVODmZmVnBMDrw3ruK+B4Bti97fgD0x\nM2tOPnMwM7MCJwczMytwcjAzswInBzMzK3ByMDOzAicHMzMrcHIwM7MCJwczMytwcjAzswInBzMz\nK3ByMDOzAicHMzMrcHIwq0LSNkkbJW2QtC7FRknqkrQlPY9McUm6RVK3pCcknZPbzpxUfoukObn4\npLT97rSuhr+WZn1zcjDr23siYmJETE7vFwCrI2I8sDq9B7iIbL708cA84DbIkgmwEDgPOBdYWE4o\nqcwVufWmDX11zOpXzxzSSyTtkvRkLvYlSdvTr6oNkqbnll2bfg09I+nCXHxainVLWpCLnynp0RS/\nW9IJg1lBs0E0A1iaXi8FLs7Fl0VmDTBC0mjgQqArIvZExF6gC5iWlp0SEWsiIoBluW2ZNYV65nO4\nA/gXsgM472sR8dV8QNJZwEzgbcDpwI8lvSUtvhW4AOgB1kpaERGbga+kbS2X9HVgLumXl1kDBfAj\nSQF8IyIWAx0RsSMtfwHoSK/HAM/n1u1JsVrxnirxAknzyM5G6OjooFQqFcr09vYyf8LBw2LVyrWa\n3t7etqhHpVapV7/JISIeljSuzu3NAJZHxMvALyR1k51OA3RHxFYAScuBGZKeAt4LfCyVWQp8CScH\na7x3RcR2Sa8HuiQ9nV8YEZESx5BKSWkxwOTJk6Ozs7NQplQqceMj+w+Lbbu0WK7VlEolqtW31bVK\nvQYyE9zVkmYD64D56bR5DLAmVyb/i6jyF9R5wKnAixFxoEr5gmq/ourJwvMnHKi5HBr7S6tVfknU\n0g51yIuI7el5l6T7yX7k7JQ0OiJ2pKahXan4duCM3OpjU2w70FkRL6X42CrlzZrG0SaH24DryU69\nrwduBD4xWDvVl2q/ourJwpdVmRa0UiN/abXKL4la2qEOZZJOBl4REb9Or6cC1wErgDnAovT8QFpl\nBdmPpeVkP3peSglkFfAPuYvQU4FrI2KPpH2SpgCPArOBfx6u+pnV46iSQ0TsLL+W9E3g++ltX7+g\n6CO+m+zi3fHp7MG/oKwZdAD3p96lxwPfiYgfSloL3CNpLvAccEkqvxKYDnQDvwEuB0hJ4HpgbSp3\nXUTsSa+vJLuedxLwYHqYNY2jSg7lU+v09kNAuSfTCuA7km4iuyA9HngMEDBe0plk//xnAh9L7bYP\nAR8GlnP4rzGzhkjXxt5eJb4bOL9KPICr+tjWEmBJlfg64OwB76zZEOk3OUi6i6zd9DRJPWT9tjsl\nTSRrVtoGfBIgIjZJugfYDBwAroqIg2k7VwOrgOOAJRGxKX3E54Hlkr4MPA7cPmi1MzOzo1JPb6VZ\nVcJ9/gOPiBuAG6rEV5KdflfGt3KoR5OZmTUB3yFtZmYFTg5mZlbg5GBmZgVODmZmVuDkYGZmBU4O\nZmZWMJCxldrKuIohNrYten+D9sTMrPF85mBmZgVODmZmVuDkYGZmBU4OZmZW4ORgZmYFTg5mZlbg\n5GBmZgVODmZmVuDkYGZmBU4OZmZW4ORgZmYF/SYHSUsk7ZL0ZC72j5KelvSEpPsljUjxcZJ+K2lD\nenw9t84kSRsldUu6RZJSfJSkLklb0vPIoaiomZnVr54zhzuAaRWxLuDsiPhL4D+Aa3PLno2Iienx\nqVz8NuAKYHx6lLe5AFgdEeOB1em9mZk1UL/JISIeBvZUxH4UEQfS2zXA2FrbkDQaOCUi1kREAMuA\ni9PiGcDS9HppLm5mZg0yGEN2fwK4O/f+TEmPA/uAL0bET4ExQE+uTE+KAXRExI70+gWgo68PkjQP\nmAfQ0dFBqVSit7eXUqlUcwfnTzhQc3k1/W1zMNVTh2bXDnWoJOk4YB2wPSI+IOlMYDlwKrAe+HhE\n/F7SiWQ/eCYBu4GPRsS2tI1rgbnAQeBvI2JVik8DbgaOA74VEYuGtXJm/RhQcpD0d8AB4M4U2gG8\nMSJ2S5oEfE/S2+rdXkSEpKixfDGwGGDy5MnR2dlJqVSis7Oz5nYvq5iroR7bLq29zcFUTx2aXTvU\noYprgKeAU9L7rwBfi4jl6XraXLLm0rnA3oh4s6SZqdxHJZ0FzATeBpwO/FjSW9K2bgUuIPuhtFbS\niojYPFwVM+vPUfdWknQZ8AHg0tRURES8HBG70+v1wLPAW4DtHN70NDbFAHamZqdy89Ouo90ns8Ei\naSzwfuBb6b2A9wL3piL5JtB80+i9wPmp/Axgefpe/ALoBs5Nj+6I2BoRvyc7G5kx9LUyq99RJYd0\nSvw54IMR8Ztc/HXpVBxJbyK78Lw1NRvtkzQlfWlmAw+k1VYAc9LrObm4WSP9E9kx/sf0/lTgxdy1\ntnzT6BjgeYC0/KVU/k/xinX6ips1jX6blSTdBXQCp0nqARaS9U46EehKPVLXpJ5J7wauk/QHsi/V\npyKifDH7SrKeTycBD6YHwCLgHklzgeeASwalZmZHSdIHgF0RsV5SZ4P3pXCdrVJvby/zJxw8LNYO\n13/a8ToWtE69+k0OETGrSvj2PsreB9zXx7J1wNlV4ruB8/vbD7Nh9E7gg5KmA68iu+ZwMzBC0vHp\n7CDfNLodOAPokXQ88FqyC9PleFl+nb7ih6l2na1SqVTixkf2HxYbzmtmQ6VNr2O1TL18h7RZhYi4\nNiLGRsQ4sgvKP4mIS4GHgA+nYvkm0HzT6IdT+UjxmZJOTD2dxgOPAWuB8ZLOlHRC+owVw1A1s7oN\nRldWs2PF54Hlkr4MPM6hM+jbgW9L6ia7J2gmQERsknQPsJmsV99VEXEQQNLVwCqyrqxLImLTsNbE\nrB9ODmY1REQJKKXXW8l6GlWW+R3wkT7WvwG4oUp8JbByEHfVbFC5WcnMzAqcHMzMrMDJwczMCpwc\nzMyswMnBzMwKnBzMzKzAycHMzAqcHMzMrMDJwczMCpwczMyswMnBzMwKnBzMzKzAycHMzAqcHMzM\nrMDJwczMCupKDpKWSNol6clcbJSkLklb0vPIFJekWyR1S3pC0jm5deak8lskzcnFJ0namNa5RWli\najMza4x6zxzuAKZVxBYAqyNiPLA6vQe4iGw6xPFkE6PfBlkyARYC55FNmLKwnFBSmSty61V+1rAb\nt+AHhz3MzI4ldSWHiHiYbPrDvBnA0vR6KXBxLr4sMmvIJmUfDVwIdEXEnojYC3QB09KyUyJiTZp3\nd1luW2Zm1gADmSa0IyJ2pNcvAB3p9Rjg+Vy5nhSrFe+pEi+QNI/sbISOjg5KpRK9vb2USqWaOzp/\nwoE6qlNbf58xEPXUodm1Qx3M7JBBmUM6IkJSDMa2+vmcxcBigMmTJ0dnZyelUonOzs6a6102CM1C\n2y6t/RkDUU8dml071MHMDhlIb6WdqUmI9LwrxbcDZ+TKjU2xWvGxVeJmZtYgA0kOK4Byj6M5wAO5\n+OzUa2kK8FJqfloFTJU0Ml2IngqsSsv2SZqSeinNzm3LzMwaoK5mJUl3AZ3AaZJ6yHodLQLukTQX\neA64JBVfCUwHuoHfAJcDRMQeSdcDa1O56yKifJH7SrIeUScBD6aHmZk1SF3JISJm9bHo/CplA7iq\nj+0sAZZUia8Dzq5nX8zMbOj5DmmzKiS9StJjkn4uaZOkv0/xMyU9mm7YvFvSCSl+YnrfnZaPy23r\n2hR/RtKFufi0FOuWtKByH8waycnBrLqXgfdGxNuBiWT35EwBvgJ8LSLeDOwF5qbyc4G9Kf61VA5J\nZwEzgbeR3dz5vyQdJ+k44Faym0bPAmalsmZNwcnBrIp0E2dvevvK9AjgvcC9KV5582f5ptB7gfNT\nB4sZwPKIeDkifkF2Le7c9OiOiK0R8XtgeSpr1hQG5T4Hs3aUft2vB95M9iv/WeDFiCjfVZm/YfNP\nN3lGxAFJLwGnpvia3Gbz61TeFHpelX0o3PhZqbe3l/kTDh4Wa4cbEtv1xspWqZeTg1kfIuIgMFHS\nCOB+4C8asA+FGz8rlUolbnxk/2Gxobxpc7i0642VrVIvNyuZ9SMiXgQeAv6KbKyw8o+q/A2bf7rJ\nMy1/LbCbI78p1KwptN2Zg0dQtcEg6XXAHyLiRUknAReQXWR+CPgw2TWCyps/5wD/npb/JA0rswL4\njqSbgNPJRh1+DBAwXtKZZElhJvCx4aqfWX/aLjmYDZLRwNJ03eEVwD0R8X1Jm4Hlkr4MPA7cnsrf\nDnxbUjfZCMYzASJik6R7gM3AAeCq1FyFpKvJRg44DlgSEZuGr3pmtTk5mFUREU8A76gS30rW06gy\n/jvgI31s6wbghirxlWQjCpg1HV9zMDOzAicHMzMrcHIwM7MCJwczMytwcjAzswInBzMzK3ByMDOz\nAicHMzMrcHIwM7OCo04Okt4qaUPusU/SZyR9SdL2XHx6bh3PiGVm1gKOeviMiHiGbIas8rj328mG\nNb6cbKasr+bLV8yIdTrwY0lvSYtvJRvYrAdYK2lFRGw+2n0zM7OBGayxlc4Hno2I57LJr6r604xY\nwC/SAGXlMWq605g1SCrPiOXkYGbWIIOVHGYCd+XeXy1pNrAOmB8RexngjFhQfVasylmV5k84UG3V\nARvKmZtaZWaoWtqhDmZ2yICTg6QTgA8C16bQbcD1ZPPtXg/cCHxioJ8D1WfFqpxV6bIhms9hKGfW\napWZoWpphzqY2SGDceZwEfCziNgJUH4GkPRN4Pvpba2ZrzwjlplZExmMrqyzyDUpSRqdW/Yh4Mn0\negUwU9KJafar8oxYa0kzYqWzkJmprJmZNciAzhwknUzWy+iTufD/lDSRrFlpW3mZZ8QyM2sdA0oO\nEbEfOLUi9vEa5Vt2Rqxqc1NvW/T+BuyJmdnQ8x3SZmZW4ORgZmYFTg5mZlbg5GBmZgVODmZmVuDk\nYGZmBU4OZhUknSHpIUmbJW2SdE2Kj5LUJWlLeh6Z4pJ0Sxpy/glJ5+S2NSeV3yJpTi4+SdLGtM4t\nqjFipVkjODmYFR0gGzDyLGAKcFUacn4BsDoixgOr03vIhpAZnx7zyMYXQ9IoYCHZQJLnAgvLCSWV\nuSK33rRhqJdZ3ZwczCpExI6I+Fl6/WvgKbIRhGcAS1OxpcDF6fUMYFlk1gAj0jAyFwJdEbEnjUzc\nBUxLy06JiDUREcCy3LbMmsJgDdlt1pYkjQPeATwKdETEjrToBaAjvR5Dcdj5Mf3Ee6rEq31+YZj6\nSr29vcyfcPCwWDsMn96uw8C3Sr2cHMz6IOnVwH3AZyJiX/6yQESEpBjqfag2TH2lUqnEjY/sPyw2\nlEPMD5d2HQa+VerlZiWzKiS9kiwx3BkR303hneVRh9PzrhTvazj6WvGxVeJmTcPJwaxC6jl0O/BU\nRNyUW7QCKPc4mgM8kIvPTr2WpgAvpeanVcBUSSPTheipwKq0bJ+kKemzZue2ZdYU3KxkVvRO4OPA\nRkkbUuwLwCLgHklzgeeAS9KylcB0oBv4DXA5QETskXQ92ZwlANdFxJ70+krgDuAk4MH0MGsaTg5m\nFSLiEaCv+w7Or1I+gKv62NYSYEmV+Drg7AHsptmQcrOSmZkV+MzBrA1VTk7liansSPnMwczMCgac\nHCRtS2PEbJC0LsUGbQwaMzMbfoN15vCeiJgYEZPT+8Ecg8bMzIbZUDUrDcoYNEO0b2Zm1o/BuCAd\nwI/SUALfSLf7D9YYNIepNs5M5Tgl8yccGIQq1WewxkdplbFWammHOpjZIYORHN4VEdslvR7okvR0\nfuFgjkFTbZyZynFKLqvopTGUBmv8mlYZa6WWdqiDmR0y4GaliNienncB95NdMxisMWjMzKwBBpQc\nJJ0s6TXl12RjxzzJII1BM5B9MzOzozfQZqUO4P40lPHxwHci4oeS1jJ4Y9CYmdkwG1ByiIitwNur\nxHczSGPQmJnZ8PPwGQPgIQrMrF15+AwzMytwcjAzswInBzMzK3ByMDOzAicHMzMrcHIwM7MCJwcz\nMytwcjAzswInBzMzK3ByMDOzAicHMzMrcHIwq0LSEkm7JD2Zi42S1CVpS3oemeKSdIukbklPSDon\nt86cVH6LpDm5+CRJG9M6tygNbWzWLJwczKq7g+I85guA1RExHlid3gNcBIxPj3nAbZAlE2AhcB7Z\nJFgLywkllbkit57nTLem4uRgVkVEPAxUzikyA1iaXi8FLs7Fl0VmDTAizYB4IdAVEXsiYi/QBUxL\ny06JiDVpGPtluW2ZNQUP2W1Wv440cyHAC2STXQGMAZ7PletJsVrxnirxAknzyM5G6OjooFQqFcr0\n9vYyf8LBmjtebb1m19vb25L73Z9WqZeTwyCqnN8BPMdDu4qIkBTD8DmLgcUAkydPjs7OzkKZUqnE\njY/sr7mdbZcW12t2pVKJavVtda1SLzcrmdVvZ2oSIj3vSvHtwBm5cmNTrFZ8bJW4WdM46uQg6QxJ\nD0naLGmTpGtS/EuStkvakB7Tc+tcm3pnPCPpwlx8Wop1S1pQ7fPMmsAKoNzjaA7wQC4+O/VamgK8\nlJqfVgFTJY1MF6KnAqvSsn2SpqReSrNz2zJrCgNpVjoAzI+In0l6DbBeUlda9rWI+Gq+sKSzgJnA\n24DTgR9LektafCtwAVnb61pJKyJi8wD2zWxAJN0FdAKnSeoh63W0CLhH0lzgOeCSVHwlMB3oBn4D\nXA4QEXskXQ+sTeWui4jyRe4ryXpEnQQ8mB5mTeOok0P69bMjvf61pKfo46JaMgNYHhEvA7+Q1E3W\nvQ+gOyK2Akhanso6OVjDRMSsPhadX6VsAFf1sZ0lwJIq8XXA2QPZR7OhNCgXpCWNA94BPAq8E7ha\n0mxgHdnZxV6yxLEmt1q+h0Zlj47z+vicQs+Nyiv/8yccGHiFBlE9vRJapfdCLe1QBzM7ZMDJQdKr\ngfuAz0TEPkm3AdcDkZ5vBD4x0M+B6j03Kq/8X1alx1Aj1dNLpFV6L9TSDnUws0MGlBwkvZIsMdwZ\nEd8FiIidueXfBL6f3vbVc4MacTMza4CB9FYScDvwVETclIuPzhX7EFAem2YFMFPSiZLOJBsy4DGy\ni3XjJZ0p6QSyi9Yrjna/zMxs4AZy5vBO4OPARkkbUuwLwCxJE8malbYBnwSIiE2S7iG70HwAuCoi\nDgJIupqs299xwJKI2DSA/TIzswEaSG+lR4BqI0murLHODcANVeIra63XyirvmvYd02bWCnyHtJmZ\nFTg5mJlZgZODmZkVODmYmVmBk4OZmRU4OZiZWYEn+zE7BngiKjtSPnMwM7MCnzk0gY3bXzpswED/\nojOzRvOZg5mZFTg5mJlZgZODmZkVODmYmVmBL0g3IY/kamaN5jMHMzMrcHIwM7MCNyu1AN/dakPB\nzZdWi88czMysoGnOHCRNA24mm0f6WxGxqMG71NT8q6/1+Zi3ZtYUyUHSccCtwAVAD7BW0oqI2NzY\nPWsdbnpqLc14zPsYsrymSA7AuUB3RGwFkLQcmAE4OQxAtS/7kfI/hyHTEsd8PceQj5H21CzJYQzw\nfO59D3BeZSFJ84B56W2vpGeA04BfDfkeDhJ9pWq4aevQx/5W07R1qOHPGvjZAznmKzX0b38Ex8iR\nasVjqh6Nrlddx32zJIe6RMRiYHE+JmldRExu0C4NCtfB+lLtmK/Urn9716uxmqW30nbgjNz7sSlm\n1q58zFtTa5bksBYYL+lMSScAM4EVDd4ns6HkY96aWlM0K0XEAUlXA6vIuvUtiYhNda5e85S7RbgO\nx5gBHvOV2vVv73o1kCKi0ftw05ypAAACzElEQVRgZmZNplmalczMrIk4OZiZWUHLJgdJ0yQ9I6lb\n0oJG7089JC2RtEvSk7nYKEldkrak55GN3Mf+SDpD0kOSNkvaJOmaFG+perSLVvwelEnaJmmjpA2S\n1qVY1eNImVtSPZ+QdE5j9/6QI/le16qHpDmp/BZJcxpRl7yWTA65oQcuAs4CZkk6q7F7VZc7gGkV\nsQXA6ogYD6xO75vZAWB+RJwFTAGuSn/7VqtHy2vh70HeeyJiYq7ff1/H0UXA+PSYB9w27Hvatzuo\n/3tdtR6SRgELyW6EPBdY2OgfWC2ZHMgNPRARvwfKQw80tYh4GNhTEZ4BLE2vlwIXD+tOHaGI2BER\nP0uvfw08RXa3b0vVo0205PegH30dRzOAZZFZA4yQNLoRO1jpCL/XfdXjQqArIvZExF6gi2LCGVat\nmhyqDT0wpkH7MlAdEbEjvX4B6GjkzhwJSeOAdwCP0sL1aGGt/j0I4EeS1qdhQqDv46jV6nqk9Wi6\n+jXFfQ6WiYiQ1BJ9iyW9GrgP+ExE7JP0p2WtVA9rqHdFxHZJrwe6JD2dX9gux1Gr1qNVzxzaaeiB\nneXT4/S8q8H70y9JryRLDHdGxHdTuOXq0QZa+nsQEdvT8y7gfrJmsr6Oo1ar65HWo+nq16rJoZ2G\nHlgBlHsmzAEeaOC+9EvZKcLtwFMRcVNuUUvVo0207PdA0smSXlN+DUwFnqTv42gFMDv19pkCvJRr\ntmlGR1qPVcBUSSPTheipKdY4EdGSD2A68B/As8DfNXp/6tznu4AdwB/I2hTnAqeS9WbYAvwYGNXo\n/eynDu8iayt+AtiQHtNbrR7t8mjF70Ha7zcBP0+PTeV97+s4AkTWM+tZYCMwudF1yNWl7u91rXoA\nnwC60+PyRtfLw2eYmVlBqzYrmZnZEHJyMDOzAicHMzMrcHIwM7MCJwczMytwcjAzswInBzMzK/j/\nGVUGM5tv3DMAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e3C0v1A-b902",
        "colab_type": "text"
      },
      "source": [
        "Interesting. We can fix the maximum length of the summary to 8 since that seems to be the majority review length.\n",
        "\n",
        "Let us understand the proportion of the length of summaries below 8"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "54bIGf-uZH0p",
        "colab_type": "code",
        "outputId": "994932d9-228c-4497-b230-1f9c5978de1b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "count = 0\n",
        "for i in data['cleaned_summary']:\n",
        "  if(len(i.split())<=8):\n",
        "    count += 1\n",
        "print(count/len(data['cleaned_summary']))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.9424907471335922\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RZyuYezqc4F5",
        "colab_type": "text"
      },
      "source": [
        "We observe that 94% of the summaries have length below 8. So, we can fix maximum length of summary to 8.\n",
        "\n",
        "Let us fix the maximum length of review to 30"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tpSJC7RUcdTY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "max_text_len=30\n",
        "max_summary_len=8"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y4pQjOkbc-FH",
        "colab_type": "text"
      },
      "source": [
        "Let us select the reviews and summaries whose length falls below or equal to max_text_len and max_summary_len\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "01gy6Tt3c6KH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cleaned_text =np.array(data['cleaned_text'])\n",
        "cleaned_summary=np.array(data['cleaned_summary'])\n",
        "\n",
        "short_text=[]\n",
        "short_summary=[]\n",
        "\n",
        "for i in range(len(cleaned_text)):\n",
        "  if(len(cleaned_summary[i].split())<=max_summary_len and len(cleaned_text[i].split())<=max_text_len):\n",
        "        short_text.append(cleaned_text[i])\n",
        "        short_summary.append(cleaned_summary[i])\n",
        "\n",
        "df=pd.DataFrame({'text':short_text,'summary':short_summary})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g21Vd5s2doLV",
        "colab_type": "text"
      },
      "source": [
        "Remember to add the **START** and **END** special tokens at the beginning and end of the summary. Here, I have chosen sostok and eostok as START and END tokens\n",
        "\n",
        "Note: Be sure that the chosen special tokens never appear in the summary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Th8GicopdA5Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['summary'] = df['summary'].apply(lambda x : 'sostok '+ x + ' eostok')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oXKkMYNdeDeL",
        "colab_type": "text"
      },
      "source": [
        "We are getting closer to the model building part. Before that, we need to split our dataset into a training and validation set. We’ll use 90% of the dataset as the training data and evaluate the performance on the remaining 10% (holdout set):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "26f2cKr8dEIy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "x_tr,x_val,y_tr,y_val=train_test_split(np.array(df['text']),np.array(df['summary']),test_size=0.1,random_state=0,shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ehrL0EFEeRv4",
        "colab_type": "text"
      },
      "source": [
        "## Preparing the Tokenizer\n",
        "A tokenizer builds the vocabulary and converts a word sequence to an integer sequence. Go ahead and build tokenizers for text and summary:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tAkadav0eV5w",
        "colab_type": "text"
      },
      "source": [
        "## Text Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yqfic_QNeGHM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.preprocessing.text import Tokenizer \n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "#prepare a tokenizer for reviews on training data\n",
        "x_tokenizer = Tokenizer() \n",
        "x_tokenizer.fit_on_texts(list(x_tr))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0LZCI7-Cfc3h",
        "colab_type": "text"
      },
      "source": [
        "## Rarewords and its Coverage\n",
        "\n",
        "Let us look at the proportion rare words and its total coverage in the entire summary\n",
        "\n",
        "Here, I am defining the threshold to be 6 which means word whose count is below 6 is considered as a rare word"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b9nU2H5NeZZD",
        "colab_type": "code",
        "outputId": "85dcc283-a609-45c9-ef0e-8ca24f728e87",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "thresh=4\n",
        "\n",
        "cnt=0\n",
        "tot_cnt=0\n",
        "freq=0\n",
        "tot_freq=0\n",
        "\n",
        "for key,value in x_tokenizer.word_counts.items():\n",
        "    tot_cnt=tot_cnt+1\n",
        "    tot_freq=tot_freq+value\n",
        "    if(value<thresh):\n",
        "        cnt=cnt+1\n",
        "        freq=freq+value\n",
        "    \n",
        "print(\"% of rare words in vocabulary:\",(cnt/tot_cnt)*100)\n",
        "print(\"Total Coverage of rare words:\",(freq/tot_freq)*100)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "% of rare words in vocabulary: 66.12339930151339\n",
            "Total Coverage of rare words: 2.953684513790566\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J3HPqFsFe_Iw",
        "colab_type": "text"
      },
      "source": [
        "Remember:\n",
        "\n",
        "**tot_cnt** gives the size of vocabulary (which means every unique words in the text)\n",
        "\n",
        "**cnt gives** me the no. of rare words whose count falls below threshold\n",
        "\n",
        "**tot_cnt** - cnt gives me the top most common words\n",
        "\n",
        "Let us define the tokenizer with top most common words for reviews.\n",
        "\n",
        "Let us define the tokenizer with top most common words for reviews."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S6tE0aATeqR8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#prepare a tokenizer for reviews on training data\n",
        "x_tokenizer = Tokenizer(num_words=tot_cnt-cnt) \n",
        "x_tokenizer.fit_on_texts(list(x_tr))\n",
        "\n",
        "#convert text sequences into integer sequences\n",
        "x_tr_seq    =   x_tokenizer.texts_to_sequences(x_tr) \n",
        "x_val_seq   =   x_tokenizer.texts_to_sequences(x_val)\n",
        "\n",
        "#padding zero upto maximum length\n",
        "x_tr    =   pad_sequences(x_tr_seq,  maxlen=max_text_len, padding='post')\n",
        "x_val   =   pad_sequences(x_val_seq, maxlen=max_text_len, padding='post')\n",
        "\n",
        "#size of vocabulary ( +1 for padding token)\n",
        "x_voc   =  x_tokenizer.num_words + 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QNsencfWgtHU",
        "colab_type": "code",
        "outputId": "bf5aaa81-2dda-4157-f044-468ab1a8249b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x_voc"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8440"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YnMtu3f6fTAO",
        "colab_type": "text"
      },
      "source": [
        "## Summary Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Qk1ZjtCfRJo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#prepare a tokenizer for reviews on training data\n",
        "y_tokenizer = Tokenizer()   \n",
        "y_tokenizer.fit_on_texts(list(y_tr))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IGUKlErkgyfB",
        "colab_type": "text"
      },
      "source": [
        "## Rarewords and its Coverage\n",
        "Let us look at the proportion rare words and its total coverage in the entire summary\n",
        "\n",
        "Here, I am defining the threshold to be 6 which means word whose count is below 6 is considered as a rare word"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LaQPPt0lgyIH",
        "colab_type": "code",
        "outputId": "82b9f406-eae8-44fe-a1f8-b104ef1e5cee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "thresh=6\n",
        "\n",
        "cnt=0\n",
        "tot_cnt=0\n",
        "freq=0\n",
        "tot_freq=0\n",
        "\n",
        "for key,value in y_tokenizer.word_counts.items():\n",
        "    tot_cnt=tot_cnt+1\n",
        "    tot_freq=tot_freq+value\n",
        "    if(value<thresh):\n",
        "        cnt=cnt+1\n",
        "        freq=freq+value\n",
        "    \n",
        "print(\"% of rare words in vocabulary:\",(cnt/tot_cnt)*100)\n",
        "print(\"Total Coverage of rare words:\",(freq/tot_freq)*100)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "% of rare words in vocabulary: 78.12740675541863\n",
            "Total Coverage of rare words: 5.3921899389571895\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d3kDM9agg5KQ",
        "colab_type": "text"
      },
      "source": [
        "Let us define the tokenizer with top most common words for summary."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_ik_4hlvft-B",
        "colab_type": "text"
      },
      "source": [
        "Here, I am deleting the rows that contain only START and END tokens"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UteuwqC5g6Dx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#prepare a tokenizer for reviews on training data\n",
        "y_tokenizer = Tokenizer(num_words=tot_cnt-cnt) \n",
        "y_tokenizer.fit_on_texts(list(y_tr))\n",
        "\n",
        "#convert text sequences into integer sequences\n",
        "y_tr_seq    =   y_tokenizer.texts_to_sequences(y_tr) \n",
        "y_val_seq   =   y_tokenizer.texts_to_sequences(y_val) \n",
        "\n",
        "#padding zero upto maximum length\n",
        "y_tr    =   pad_sequences(y_tr_seq, maxlen=max_summary_len, padding='post')\n",
        "y_val   =   pad_sequences(y_val_seq, maxlen=max_summary_len, padding='post')\n",
        "\n",
        "#size of vocabulary\n",
        "y_voc  =   y_tokenizer.num_words +1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kagyaTnLfnX7",
        "colab_type": "text"
      },
      "source": [
        "Let us check whether word count of start token is equal to length of the training data\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3l8M_vXHfNrI",
        "colab_type": "code",
        "outputId": "ee694d06-25e9-4f9a-fcb2-2ec1045ea809",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "y_tokenizer.word_counts['sostok'],len(y_tr)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(42453, 42453)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cDsdp_fdhPFJ",
        "colab_type": "text"
      },
      "source": [
        "Here, I am deleting the rows that contain only **START** and **END** tokens\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2V_hEzWHftNM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ind=[]\n",
        "for i in range(len(y_tr)):\n",
        "    cnt=0\n",
        "    for j in y_tr[i]:\n",
        "        if j!=0:\n",
        "            cnt=cnt+1\n",
        "    if(cnt==2):\n",
        "        ind.append(i)\n",
        "\n",
        "y_tr=np.delete(y_tr,ind, axis=0)\n",
        "x_tr=np.delete(x_tr,ind, axis=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YkiFhWivf005",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ind=[]\n",
        "for i in range(len(y_val)):\n",
        "    cnt=0\n",
        "    for j in y_val[i]:\n",
        "        if j!=0:\n",
        "            cnt=cnt+1\n",
        "    if(cnt==2):\n",
        "        ind.append(i)\n",
        "\n",
        "y_val=np.delete(y_val,ind, axis=0)\n",
        "x_val=np.delete(x_val,ind, axis=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "py3hsshMgJUF",
        "colab_type": "text"
      },
      "source": [
        "## Model building\n",
        "We are finally at the model building part. But before we do that, we need to familiarize ourselves with a few terms which are required prior to building the model.\n",
        "\n",
        "**Return Sequences** = True: When the return sequences parameter is set to True, LSTM produces the hidden state and cell state for every timestep\n",
        "\n",
        "**Return State** = True: When return state = True, LSTM produces the hidden state and cell state of the last timestep only\n",
        "\n",
        "**Initial State: **This is used to initialize the internal states of the LSTM for the first timestep\n",
        "\n",
        "**Stacked LSTM**:   Stacked LSTM has multiple layers of LSTM stacked on top of each other. This leads to a better representation of the sequence. I encourage you to experiment with the multiple layers of the LSTM stacked on top of each other (it’s a great way to learn this)\n",
        "\n",
        "Here, we are building a 3 stacked LSTM for the encoder:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l_erB3n2f4mm",
        "colab_type": "code",
        "outputId": "b1922b81-3c92-4ca3-ca82-c6f8b5339756",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 836
        }
      },
      "source": [
        "from keras import backend as K \n",
        "K.clear_session()\n",
        "\n",
        "latent_dim = 300\n",
        "embedding_dim=100\n",
        "\n",
        "# Encoder\n",
        "encoder_inputs = Input(shape=(max_text_len,))\n",
        "\n",
        "#embedding layer\n",
        "enc_emb =  Embedding(x_voc, embedding_dim,trainable=True)(encoder_inputs)\n",
        "\n",
        "#encoder lstm 1\n",
        "encoder_lstm1 = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.4,recurrent_dropout=0.4)\n",
        "encoder_output1, state_h1, state_c1 = encoder_lstm1(enc_emb)\n",
        "\n",
        "#encoder lstm 2\n",
        "encoder_lstm2 = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.4,recurrent_dropout=0.4)\n",
        "encoder_output2, state_h2, state_c2 = encoder_lstm2(encoder_output1)\n",
        "\n",
        "#encoder lstm 3\n",
        "encoder_lstm3=LSTM(latent_dim, return_state=True, return_sequences=True,dropout=0.4,recurrent_dropout=0.4)\n",
        "encoder_outputs, state_h, state_c= encoder_lstm3(encoder_output2)\n",
        "\n",
        "# Set up the decoder, using `encoder_states` as initial state.\n",
        "decoder_inputs = Input(shape=(None,))\n",
        "\n",
        "#embedding layer\n",
        "dec_emb_layer = Embedding(y_voc, embedding_dim,trainable=True)\n",
        "dec_emb = dec_emb_layer(decoder_inputs)\n",
        "\n",
        "decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True,dropout=0.4,recurrent_dropout=0.2)\n",
        "decoder_outputs,decoder_fwd_state, decoder_back_state = decoder_lstm(dec_emb,initial_state=[state_h, state_c])\n",
        "\n",
        "# Attention layer\n",
        "attn_layer = AttentionLayer(name='attention_layer')\n",
        "attn_out, attn_states = attn_layer([encoder_outputs, decoder_outputs])\n",
        "\n",
        "# Concat attention input and decoder LSTM output\n",
        "decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attn_out])\n",
        "\n",
        "#dense layer\n",
        "decoder_dense =  TimeDistributed(Dense(y_voc, activation='softmax'))\n",
        "decoder_outputs = decoder_dense(decoder_concat_input)\n",
        "\n",
        "# Define the model \n",
        "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING: Logging before flag parsing goes to stderr.\n",
            "W0622 10:54:09.226793 140120804251520 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:95: The name tf.reset_default_graph is deprecated. Please use tf.compat.v1.reset_default_graph instead.\n",
            "\n",
            "W0622 10:54:09.229068 140120804251520 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:98: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
            "\n",
            "W0622 10:54:09.274516 140120804251520 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:102: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n",
            "W0622 10:54:09.282836 140120804251520 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "W0622 10:54:09.302339 140120804251520 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 30)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding (Embedding)           (None, 30, 100)      844000      input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lstm (LSTM)                     [(None, 30, 300), (N 481200      embedding[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "input_2 (InputLayer)            [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "lstm_1 (LSTM)                   [(None, 30, 300), (N 721200      lstm[0][0]                       \n",
            "__________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)         (None, None, 100)    198900      input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lstm_2 (LSTM)                   [(None, 30, 300), (N 721200      lstm_1[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "lstm_3 (LSTM)                   [(None, None, 300),  481200      embedding_1[0][0]                \n",
            "                                                                 lstm_2[0][1]                     \n",
            "                                                                 lstm_2[0][2]                     \n",
            "__________________________________________________________________________________________________\n",
            "attention_layer (AttentionLayer ((None, None, 300),  180300      lstm_2[0][0]                     \n",
            "                                                                 lstm_3[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "concat_layer (Concatenate)      (None, None, 600)    0           lstm_3[0][0]                     \n",
            "                                                                 attention_layer[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "time_distributed (TimeDistribut (None, None, 1989)   1195389     concat_layer[0][0]               \n",
            "==================================================================================================\n",
            "Total params: 4,823,389\n",
            "Trainable params: 4,823,389\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "odNBDquMhYVr",
        "colab_type": "text"
      },
      "source": [
        "I am using sparse categorical cross-entropy as the loss function since it converts the integer sequence to a one-hot vector on the fly. This overcomes any memory issues."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vKjlUQQ7fUy6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZUCJf3m4hdhB",
        "colab_type": "text"
      },
      "source": [
        "Remember the concept of early stopping? It is used to stop training the neural network at the right time by monitoring a user-specified metric. Here, I am monitoring the validation loss (val_loss). Our model will stop training once the validation loss increases:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kIaaynevhbFr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=2)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-KVsU8U7hgh3",
        "colab_type": "text"
      },
      "source": [
        "We’ll train the model on a batch size of 128 and validate it on the holdout set (which is 10% of our dataset):\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3RJxyGoKhe34",
        "colab_type": "code",
        "outputId": "7e29d7ad-b229-493f-bfcf-ef7655eeeeaf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 598
        }
      },
      "source": [
        "history=model.fit([x_tr,y_tr[:,:-1]], y_tr.reshape(y_tr.shape[0],y_tr.shape[1], 1)[:,1:] ,epochs=50,callbacks=[es],batch_size=128, validation_data=([x_val,y_val[:,:-1]], y_val.reshape(y_val.shape[0],y_val.shape[1], 1)[:,1:]))\n"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 41346 samples, validate on 4588 samples\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0622 10:54:12.029613 140120804251520 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n",
            "41346/41346 [==============================] - 69s 2ms/sample - loss: 2.8230 - val_loss: 2.5982\n",
            "Epoch 2/50\n",
            "41346/41346 [==============================] - 62s 2ms/sample - loss: 2.5061 - val_loss: 2.4258\n",
            "Epoch 3/50\n",
            "41346/41346 [==============================] - 62s 2ms/sample - loss: 2.3646 - val_loss: 2.3285\n",
            "Epoch 4/50\n",
            "41346/41346 [==============================] - 62s 1ms/sample - loss: 2.2624 - val_loss: 2.2504\n",
            "Epoch 5/50\n",
            "41346/41346 [==============================] - 62s 2ms/sample - loss: 2.1913 - val_loss: 2.2002\n",
            "Epoch 6/50\n",
            "41346/41346 [==============================] - 62s 1ms/sample - loss: 2.1396 - val_loss: 2.1655\n",
            "Epoch 7/50\n",
            "41346/41346 [==============================] - 62s 1ms/sample - loss: 2.0957 - val_loss: 2.1457\n",
            "Epoch 8/50\n",
            "41346/41346 [==============================] - 62s 1ms/sample - loss: 2.0586 - val_loss: 2.1231\n",
            "Epoch 9/50\n",
            "41346/41346 [==============================] - 62s 2ms/sample - loss: 2.0255 - val_loss: 2.1006\n",
            "Epoch 10/50\n",
            "41346/41346 [==============================] - 62s 1ms/sample - loss: 1.9966 - val_loss: 2.1057\n",
            "Epoch 11/50\n",
            "41346/41346 [==============================] - 61s 1ms/sample - loss: 1.9693 - val_loss: 2.0783\n",
            "Epoch 12/50\n",
            "41346/41346 [==============================] - 61s 1ms/sample - loss: 1.9437 - val_loss: 2.0639\n",
            "Epoch 13/50\n",
            "41346/41346 [==============================] - 61s 1ms/sample - loss: 1.9203 - val_loss: 2.0770\n",
            "Epoch 14/50\n",
            "41346/41346 [==============================] - 62s 2ms/sample - loss: 1.8982 - val_loss: 2.0681\n",
            "Epoch 00014: early stopping\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PF8g_8wQh_yf",
        "colab_type": "text"
      },
      "source": [
        "## Understanding the Diagnostic plot\n",
        "\n",
        "Now, we will plot a few diagnostic plots to understand the behavior of the model over time:\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ACxG_iPZhixv",
        "colab_type": "code",
        "outputId": "15d0df29-7489-4908-d917-93763b1ccc24",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        }
      },
      "source": [
        "from matplotlib import pyplot\n",
        "pyplot.plot(history.history['loss'], label='train')\n",
        "pyplot.plot(history.history['val_loss'], label='test')\n",
        "pyplot.legend()\n",
        "pyplot.show()"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4VdW9//H3NzMZyJyQkYQZJGFG\nQIoooszaa+tIa9UWrdbqvVarvR1vb1t721r11zpVrbYqTtgyiAooKKMMIYQhgYQxISGQhISQEDKt\n3x/7AAFCchJOsnNOvq/nyZOTc9be55s88MnK2muvJcYYlFJKeRYvuwtQSinlehruSinlgTTclVLK\nA2m4K6WUB9JwV0opD6ThrpRSHkjDXSmlPJCGu1JKeSANd6WU8kA+dr1xVFSUSUlJsevtlVLKLW3Z\nsqXEGBPdWjvbwj0lJYXNmzfb9fZKKeWWROSgM+10WEYppTyQhrtSSnkgDXellPJAto25K6VUe9TV\n1VFQUEBNTY3dpXSogIAAEhMT8fX1bdfxGu5KKbdSUFBASEgIKSkpiIjd5XQIYwylpaUUFBSQmpra\nrnPosIxSyq3U1NQQGRnpscEOICJERkZe1l8nGu5KKbfjycF+xuV+j24X7nlHK/mfxbuorW+0uxSl\nlOqy3C7c88tO8dra/Xy555jdpSiluqHy8nKef/75Nh83Y8YMysvLO6Ci5rUa7iKSJCIrRWSXiOwU\nkYebaRMqIotFZJujzd0dUy5c1S+KsEBfFmcVdtRbKKXUJV0q3Ovr61s8bunSpYSFhXVUWRdxZrZM\nPfCoMSZDREKALSKy3Bizq0mbB4FdxpjZIhIN7BaRt4wxta4u2M/Hi+lD41iYeZhTtQ308PN29Vso\npdQlPfHEE+zdu5fhw4fj6+tLQEAA4eHh5OTksGfPHm666Sby8/Opqanh4YcfZt68ecC5JVdOnjzJ\n9OnTmThxIuvWrSMhIYGFCxfSo0cPl9bZargbY4qAIsfjShHJBhKApuFugBCxrgAEA2VYvxQ6xOxh\ncczfeIjPcoqZlR7fUW+jlOrifrV4J7sKT7j0nEPie/KL2Vdc8vWnnnqKHTt2kJmZyapVq5g5cyY7\nduw4O2XxtddeIyIiglOnTjFmzBhuvvlmIiMjzztHbm4u8+fP529/+xu33HILCxYsYO7cuS79Pto0\n5i4iKcAI4KsLXvoLMBgoBLYDDxtjOuyK55WpkcSE+LMoU4dmlFL2Gjt27Hlz0Z977jmGDRvGuHHj\nyM/PJzc396JjUlNTGT58OACjRo3iwIEDLq/L6ZuYRCQYWAA8Yoy58FflDUAmcC3QF1guIqsvbCci\n84B5AMnJye0u2ttLmJkex1sbDnGipo6eAe27g0sp5d5a6mF3lqCgoLOPV61axYoVK1i/fj2BgYFM\nnjy52bnq/v7+Zx97e3tz6tQpl9flVM9dRHyxgv0tY8yHzTS5G/jQWPKA/cCgCxsZY142xow2xoyO\njm51OeIWzRkWT21DI5/uOHJZ51FKqbYICQmhsrKy2dcqKioIDw8nMDCQnJwcNmzY0MnVnePMbBkB\nXgWyjTFPX6LZIWCKo30sMBDY56oimzM8KYykiB4szirqyLdRSqnzREZGctVVVzF06FAee+yx816b\nNm0a9fX1DB48mCeeeIJx48bZVCWIMablBiITgdVYY+lnxtF/AiQDGGNeFJF44HUgDhDgKWPMmy2d\nd/To0eZyN+v4v09yeOnLfWz8yRQig/1bP0Ap5fays7MZPHiw3WV0iua+VxHZYowZ3dqxzsyWWYMV\n2C21KQSub+1crjZ7WDzPr9rL0h1H+Na43p399kop1WW53R2qTQ3qFUL/mGAW66wZpZQ6j1uHu4gw\ne1g8Gw+UUVTh+qvNSinlrtw63MEamgFYsk0vrCql1BluH+6pUUGkJYTqWjNKKdWE24c7WHPeswoq\n2F9SZXcpSinVJXhEuM9MjwNgyTbtvSulOlZ7l/wFeOaZZ6iurnZxRc3ziHCPD+vB2JQIFm0rpLV5\n+0opdTncJdw9ZoPs2cPi+NnCnewurmRQr552l6OU8lBNl/ydOnUqMTExvPfee5w+fZqvf/3r/OpX\nv6KqqopbbrmFgoICGhoa+NnPfkZxcTGFhYVcc801REVFsXLlyg6t02PCfXpaHL9cvItFmYUMmqbh\nrlS38PETcGS7a8/ZKw2mP3XJl5su+bts2TI++OADNm7ciDGGOXPm8OWXX3Ls2DHi4+P56KOPAGvN\nmdDQUJ5++mlWrlxJVFSUa2tuhkcMywBEBfszoW8ki7N0aEYp1TmWLVvGsmXLGDFiBCNHjiQnJ4fc\n3FzS0tJYvnw5P/7xj1m9ejWhoaGdXpvH9NzBmjXz2AdZZOaXMyI53O5ylFIdrYUedmcwxvDkk09y\n3333XfRaRkYGS5cu5ac//SlTpkzh5z//eafW5jE9d4Drr+iFn7cXi/WGJqVUB2m65O8NN9zAa6+9\nxsmTJwE4fPgwR48epbCwkMDAQObOnctjjz1GRkbGRcd2NI/quYf28GXywGiWZBXy3zMH4+3V4npn\nSinVZk2X/J0+fTp33HEH48ePByA4OJg333yTvLw8HnvsMby8vPD19eWFF14AYN68eUybNo34+PgO\nv6Da6pK/HcUVS/42Z/G2Qh6av5X53xvH+L6RrR+glHIruuSvc0v+etSwDMCUwTEE+nmzSG9oUkp1\nYx4X7oF+Plw3OJaPdxRR19Bhe3QrpVSX5nHhDtasmfLqOtbklthdilKqA3SH6c6X+z16ZLh/bUAU\nPQN8WKxDM0p5nICAAEpLSz064I0xlJaWEhAQ0O5zeNRsmTP8fbyZPjSOJVmF1NQ1EODrbXdJSikX\nSUxMpKCggGPHjtldSocKCAggMTGx3cd7ZLiDtYnHu5vzWZlzlOlpcXaXo5RyEV9fX1JTU+0uo8vz\nyGEZgPF9I4kK9tdZM0qpbsljw93bS5iZ1ovPc45SWVNndzlKKdWpPDbcAeYMj+d0fSPLdxXbXYpS\nSnUqjw73EUnhJIT10FkzSqlux6PD3ctLmDUsjtW5JRyvqrW7HKWU6jQeHe4As9PjqW80fLzjiN2l\nKKVUp/H4cL8ivid9ooJYtO2w3aUopVSn8fhwFxFmD4vnq/1lFJ+osbscpZTqFB4f7mDd0GQMLMnS\nTTyUUt1Dtwj3fjHBDInrqbNmlFLdRrcId7B675n55eSXVdtdilJKdbhuE+6z0q31ZXQ5AqVUd9Bt\nwj0pIpCRyWE6NKOU6ha6TbiDtYlHzpFKcos7Z/dxpZSyS6vhLiJJIrJSRHaJyE4RefgS7SaLSKaj\nzReuL/XyzUiPw0vQ3rtSyuM503OvBx41xgwBxgEPisiQpg1EJAx4HphjjLkC+KbLK3WBmJAAxveN\nZNG2Qo/exUUppVoNd2NMkTEmw/G4EsgGEi5odgfwoTHmkKPdUVcX6iqz0+M5UFrNjsMn7C5FKaU6\nTJvG3EUkBRgBfHXBSwOAcBFZJSJbROTbrimvGZVHYPkvoKG+XYdPG9oLX2/R5QiUUh7N6XAXkWBg\nAfCIMebCbq8PMAqYCdwA/ExEBjRzjnkisllENrd7/8ND62HtM/DVi+06PCzQj0n9o1mSVURjow7N\nKKU8k1PhLiK+WMH+ljHmw2aaFACfGmOqjDElwJfAsAsbGWNeNsaMNsaMjo6Obl/FQ26CAdNg5W/g\n+IF2nWLO8HiKKmrYfPB4+2pQSqkuzpnZMgK8CmQbY56+RLOFwEQR8RGRQOBKrLF51xOBmX8C8YKP\nHoV2XBi9bnAsAb5eOmtGKeWxnOm5XwV8C7jWMdUxU0RmiMj9InI/gDEmG/gEyAI2Aq8YY3Z0WNWh\niTDl55C3ArZ/0ObDg/x9mDI4lqXbi6hvaOyAApVSyl4+rTUwxqwBxIl2fwD+4IqinDLmu5D1Hnzy\nBPSbAoERbTp8dno8H2UVsW5vKZMGtHOISCmluij3vUPVyxtmPws15bDsp20+fPLAaEL8fXStGaWU\nR3LfcAfoNRQm/BAy34J9q9p0aICvN9df0YtPdxzhdH1Dx9SnlFI2ce9wB7j6cYjoA4sfgbpTbTp0\nzvB4Kk/Xs2p3O6dlKqVUF+X+4e7bwxqeOb4fvvh9mw6d0DeSiCA/nTWjlPI47h/uAKmTYPhcWPsc\nHHF+ko6vtxcz0nqxIruYqtPtu+NVKaW6Is8Id4Drfw09wmHRQ9Do/Bj67PR4auoaWZFd3IHFKaVU\n5/KccA+MgOm/h8IM2Piy04eNSYmgV88AHZpRSnkUzwl3gKE3Q7+p8NmvoTzfqUO8vIRZ6XF8secY\nFdV1HVygUkp1Ds8KdxGY9TRg2rQ0wZzh8dQ1GD7ZWdSx9SmlVCfxrHAHCEuGa38KuZ/Czn85dUha\nQii9IwNZvE3DXSnlGTwv3AGuvB/iR8DHP4ZTra/8KCLMGRbPur0lHK2s6YQClVKqY3lmuHt5w+zn\noLoUlv/cqUNmD4un0cDH2490cHFKKdXxPDPcAeLSYcIPIOMfcGBNq80HxIYwqFeIrjWjlPIInhvu\nAFc/AeEpsPhhqGt9uGX2sHi2HDxOwfHqjq9NKaU6kGeHu18gzPozlObB6j+22nx2ejwAC7bo/qpK\nKffm2eEO0PdaGHY7rPkzFO9qsWlyZCBTh8Tyl5W5ZOaXd1KBSinlep4f7gDX/wYCQmHxD6Gx5Z2X\n/vCNdGJ7BvDAm1soq6rtpAKVUsq1uke4B0XCDb+Dgk2w+dUWm4YF+vHCnaMoqarl4Xe20tDY9j1a\nlVLKbt0j3AHSb4E+18CKX0FFy2PqaYmh/M+cK1idW8KzK/Z0UoFKKeU63SfcRayLq431sPRHrS5N\ncOuYJL45KpHnPs/j8xxdMVIp5V66T7gDRKTCNT+B3Ushe1GLTUWEX980lCFxPfnPd7eRX6bTI5VS\n7qN7hTvAuAegVzosfRxOtTwjJsDXmxfnjsIYw/ff2kJNne61qpRyD90v3L19YM5zUHUUVvyy1ebJ\nkYE8fctwdhw+wS8X7ez4+pRSygW6X7iDtajYuAdgy9/h4PpWm183JJYHr+nLO5vyeW+Tc+vEK6WU\nnbpnuIM19h6abM19rz/davP/mjqQq/pF8rOFO9hxuKITClRKqfbrvuHuF2TNninZA6ufbrW5t5fw\n7G0jCA/044G3MnTXJqVUl9Z9wx2g/3WQ9k1Y/Sc4trvV5lHB/vz1zpEUVZzi0fczadQbnJRSXVT3\nDnew7lz1D4ZFrS9NADCqdzg/nTmEFdlHeeGLvZ1QoFJKtZ2Ge3C0tfZM/gbIeN2pQ749vjdzhsXz\np2W7WZNb0rH1KaVUO2i4Awy/A1InwfJfwInW91EVEX73H2n0jQ7mh+9spajiVCcUqZRSztNwB8fS\nBM9AQy18/LhThwT5+/DC3FGcrmvggbcyqK1vfUhHKaU6i4b7GZF94eofW8sS7Fjg1CH9YoL5v28M\nY+uhcn67NLuDC1RKKedpuDc14SFIGA0Lvgdb33TqkJnpcdw7MZXX1x1gYabu4KSU6ho03Jvy9oVv\n/xv6XA0LH4Qv/q/V1SMBnpg+iDEp4TyxYDt7iis7oVCllGpZq+EuIkkislJEdonIThF5uIW2Y0Sk\nXkS+4doyO5F/CNz+rrU138rfWJtrN9S3eIivtxd/uWMkQf4+3P/PLVTW6A1OSil7OdNzrwceNcYM\nAcYBD4rIkAsbiYg38HtgmWtLtIGPH9z0AnztUch4A969E2qrWjwktmcAf7ljBAfLqvnxgiyMEz1+\npZTqKK2GuzGmyBiT4XhcCWQDCc00fQhYABx1aYV2EYEpP4eZT0PuMnhjNlS1PKd9XJ9IHr9hIEu3\nH+HVNfs7qVCllLpYm8bcRSQFGAF8dcHzCcDXgRdcVViXMeZeuPVNKN4Jr06Fsn0tNp83qQ/XD4nl\ndx/nsHF/WScVqZRS53M63EUkGKtn/ogx5sQFLz8D/NgY0+JkbxGZJyKbRWTzsWPH2l6tXQbNhLsW\nW5t7vDIVDm+5ZFMR4Y+3DCM5IpAfvJ3B0cqaTixUKaUsToW7iPhiBftbxpgPm2kyGnhHRA4A3wCe\nF5GbLmxkjHnZGDPaGDM6Ojr6Msq2QdJYuHcZ+AXC67Ngz6UvLfQM8OWFuSM5UVPHD97eSn2D3uCk\nlOpczsyWEeBVINsY0+zauMaYVGNMijEmBfgAeMAY82+XVtoVRPWHe1dAZD+Yfxtk/POSTQf16snv\n/iONjfvL+MOnra84qZRSruRMz/0q4FvAtSKS6fiYISL3i8j9HVxf1xMSC3cvhT6TYdEPYNXvLzkX\n/usjEpk7LpmXvtzHJztaX7NGKaVcxae1BsaYNYA4e0JjzHcupyC34B8Cd7xrLRO86rdw4rA1q8b7\n4h/nz2YNYXtBBT96P4sBsSH0iQ62oWClVHejd6i2l7cv3PQ8THrMmgv/zh3NzoX39/Hm+bmj8PUW\nvv9mBtW1Ld8QpZRSrqDhfjlE4NqfWtv15S23LrSevHgWUEJYD569bQR7jlby5IfbdQcnpVSH03B3\nhdH3wK1vwdFsay586cU7NE0aEM2Prh/IwsxCfvT+Np1Bo5TqUBrurjJoBty1CGoq4NXrm50L/8Dk\nvjw6dQAfbj3M/W9mUFPXYEOhSqnuQMPdlZLGwr3LwS/IMRf+0/NeFhEemtKfX825ghXZxdz9902c\nPK1j8Eop19Nwd7WoflbARw2A+bdDxj8uanLXhBT+fOswNh4o486/beB4Va0NhSqlPJmGe0cIiYXv\nfAR9r4FFD8HK3100F/7rIxJ5ae4oso9UcstL6zlSocsUKKVcR8O9o/gHw+3vwPA74YunrJC/YF34\n64bE8sbdYyksP8U3XlzHwdKWlxVWSilnabh3JG9fuPGvMOlx2PpPeOf2i+bCj+8byfx546g6Xc83\nXlxPdtGFa7IppVTbabh3NBG49r8dc+FXwN9nXLRscHpiGO/dNx5vEW59aT1bDh63qVillKfQcO8s\no++B2+ZD2X548WuQ+fZ54/D9Y0N4//7xhAf5MfeVr1id60ZLIiuluhwN9840cBp8fy3EDYN/fx8+\nuAdOneulJ0UE8v794+kdGcg9r2/i4+262JhSqn003DtbWJK18ceUn0P2InhhIhxYe/blmJAA3p03\nnrSEUB58O4P3NuXbWKxSyl1puNvBy9vafPveZdZm3K/PhM/+BxrqAAgN9OXN717JVf2ieHxBFq+s\nbnlrP6WUupCGu50SRsF9q2HEnbD6T9ayBY51aQL9fHjlrtHMSOvF/36UzZ+W7cZcYt14pZS6kIa7\n3fyDremS33zDmkXz4tesHZ6Mwd/Hm/93+0huHZ3E//s8j18u2qkrSiqlnKLh3lVccZN1sTVhpLXD\n0/t3QXUZ3l7CUzenMW9SH95Yf5BH399Gna4oqZRqhYZ7VxKaCN9eCNf9EnI+ghcnwv7ViAhPTh/E\nYzcM5F9bD/P9N7foipJKqRZpuHc1Xt4w8T/huyvAtwe8MRuW/wJpqOPBa/rx65uG8lnOUe56bSOV\nNXV2V6uU6qI03Luq+BFw35cw8tuw9hlrE5CSXL41rjfP3DqcLQePc+crX1GmK0oqpZqh4d6V+QXB\nnOfg1jeh/CC8NAm2vM6Nw+J5+duj2O1YUbKo4pTdlSqluhgNd3cweDZ8fx0kjoHFD8O7c7k22Zc3\n7hnLkYoavvHCeg6U6IqSSqlzNNzdRc94+Na/YeqvrR2eXpjAOLYz/3vjOFXXwDdeXM+uQl1RUill\n0XB3J15ecNUPrYutfsHwj5tI2/VH3vvuKHy9hdteXs/nOcV2V6mU6gI03N1R/HDrYuuo78C65+i3\n8Eb+dUsUcaE9uOf1zTzyzla90KpUN6fh7q78AmH2M3Db21BRQK/5N7BkQi4PX9uPJVlFTH36CxZv\nK9QlC5TqpjTc3d2gmdbF1uRx+H78X/znvnv5cvoxksL8eGj+Vr73jy26P6tS3ZCGuyfoGQdzP4Qb\nn4f608R/9kP+Vf8gb6dtZUtuPlOf/oL5Gw9pL16pbkTs+g8/evRos3nzZlve26M1NkLup7D2WTi0\nngb/MBb6TuO3JVfTv08fnro5jd6RQXZXqZRqJxHZYowZ3Wo7DXcPlr8J1j2LyV5Co5cvHzZ8jdca\nZ3Lz9ddw91WpeHuJ3RUqpdpIw12dU5IH6/+CyXwb01DL8oZRfB55G/fcdhsDe4XYXZ1Sqg003NXF\nTh7FfPUSdRtexq/uBJsbB1J4xfeY9h934+frY3d1SiknaLirS6utomrD69R8+SyR9cXkeyXAhIdI\nuvpu8A2wuzqlVAucDXedLdMd+QURNOlBIp/cxfZxT1Nt/Eha8wQnfz+YulV/hFPH7a5QKXWZWg13\nEUkSkZUisktEdorIw820uVNEskRku4isE5FhHVOucilvH9Km3Uvc4xt5tc8zbDmdiO+qX9PwpyHw\nyZNQnm93hUqpdmp1WEZE4oA4Y0yGiIQAW4CbjDG7mrSZAGQbY46LyHTgl8aYK1s6rw7LdD3r9pbw\nyvuLmFW1gBu91+MlIENvttaz6ZVmd3lKKVw4LGOMKTLGZDgeVwLZQMIFbdYZY878Lb8BSGx7ycpu\nE/pG8df/uouc8X/k6tN/5h2ZQX22Y7u/f34ddn8C9aftLlMp5YQ2XVAVkRTgS2CoMabZ9WVF5EfA\nIGPMd5t5bR4wDyA5OXnUwYMH21Gy6gzb8sv58YIsCo8U8b+Jm5h1aiFeVUfBLwT6T4XBs6DfVAjo\naXepSnUrLp8tIyLBwBfAb4wxH16izTXA88BEY0xpS+fTYZmur7a+kRdW7eUvK3OJ8Ic/jDrOxPoN\neO1eCtUl4O0HfSbDoFkwcAYER9tdslIez6XhLiK+wBLgU2PM05dokw78C5hujNnT2jk13N3HnuJK\nHv8gi8z8chLCenD/pN7cEluEf+5SyFkM5YdAvCBpnNWjHzQTwlPsLlspj+SycBcRAd4Ayowxj1yi\nTTLwOfBtY8w6ZwrUcHcvjY2Gz3OO8vyqPDIOlRMZ5Mc9E1OZe2UyoRU5kLMEcj6C4h3WAbFpjqCf\nBbFXgOhSB0q5givDfSKwGtgONDqe/gmQDGCMeVFEXgFuBs4Mote39uYa7u7JGMPG/WU8v2ovX+w5\nRrC/D3eOS+beianEhARA2T4r5LOXQP5XgLF68YNmWXvBJo61dpRSSrWL3qGqOtzOwgpeWLWXpduL\n8PH24pujErlvUl+SIwOtBpXFsHup1avf9wU01kFQDAyaAYNmQ+ok8PGz95tQys1ouKtOc6Ckipe+\n3MuCLYepb2xkVno835/cl8FxTWbS1FRA7nLIXgx5K6D2JPj3tGbeDJplffbXRcyUao2Gu+p0xSdq\neHXNft7acJCq2gauHRTDA5P7Mjol4vyGdTWw/wsr6HcvhepS8PKF6IEQO9Qao4+9wrpxKjjGnm9G\nqS5Kw13ZpqK6jn+sP8Df1x2grKqWMSnhPDC5H5MHRiMXXlhtbIBDGyBvORzZYV2QrSw693pQtCPs\nh54L/uiB4OPfqd+TUl2FhruyXXVtPe9uyudvX+6jsKKGQb1C+P7kvsxMi8PHu4WLqlWlcHQnFO88\nF/hHs6HBcXeslw9EDbg49EN66awc5fE03FWXUdfQyMLMQl78Yi95R0+SHBHIfVf34eaRiQT4ejt3\nkoZ6KNtrBf3Z0N8JJwrOtQmMbKaXP0iXMVYeRcNddTmNjYbl2cU8v2ov2/LLiQ7x596Jqdx5ZTIh\nAb7tO+mp41bIF++0gv+Io5dff8p6Xbwhqj/0SoeEkRA/wnrsF+i6b0ypTqThrrosYwzr95by/Kq9\nrMkrISTAh2+P783ccb2JC+1x+W/Q2ABl+x29fEfgF2WeG8sXb4gZbAV9wkiIH2n18r3b+QtGqU6k\n4a7cQlZBOS+s2ssnO48gwLWDYrh9bDKTB8a4fgPvE0VQuBUKM+BwhvX5zMYk3v7Qa6gV9GcCP6o/\neDk5bKRUJ9FwV24lv6ya+RsP8d7mAkpOniY+NIBbxiRx65gk1/Tmm2MMHD9ghXzhVji81erh1560\nXvcLhrjhkDDC6uXHj7TuttWLtspGGu7KLdU1NLJiVzFvbzzE6twSvASuHRTLHVcmcfWADujNX6ix\nAUpyz+/dH9kODbXW6z0izh/OSRhpzdJRqpNouCu3d6i0mnc2nd+bv3VMMreOSaJXaCfOgKmvhaO7\nmgT+VuuirWmwXg+JcwR9kx5+YETL51SqnTTclceorW9kRXYx8y/ozd95ZTKTBkR3fG++2aKqrR59\n0x5+ad6518NTzgV9/AiIH67LKyiX0HBXHulgaRXvbMrn/c35lJysJSGsB7c6xuZje9o8n72mAgoz\nm1y03QoVhxwvinWB9kzYJ4y0llfw7aDrCcpjabgrj1Zb38jyXVZvfk1eCd5ewrWDYrjjymQm9bep\nN9+cqhJH2G8918M/WWy9Jt4QM+T84ZyYIR27UmZ9rXXBuLbK+vAPhlDd8tidaLirbuNAybnefGmV\n1Zu/bUwSt3SF3nxzThSeG7s/M1OnuSmZZ4ZzfALOD+TzPk6e/7iu+hKvOT4a6y6uJzQZUiae+wjv\n3bk/D9UmGu6q26mtb2TZriPM33iItXmleHsJUwbFcHtX681fqLUpma0RL2vapl+Q9eEbeP7XZx8H\nXvB1EJw8BgfXwIG1cKrMOp+GfZem4a66tf0lVbyz6RAfbC6gtKqWmBB/ZqTFMXtYHCOSwvHqqkF/\nRmMjlOZCURaYxmaCusmHT8Dlz71vbIRj2XBgzbkPDfsuScNdKeB0fQPLdxWzKLOQVXuOUVvfSHxo\nADPT45iVHk96YujFyxArR9jnOIJ+NRxca627Dxr2NtNwV+oClTV1LN9VzJKsIlbnHqOuwZAU0YOZ\nafHMSo/jivieGvSX0mLYJ50f9mG99S7eDqThrlQLKqrr+HTXEZZkFbE2r4SGRkNqVBCz0uOYmR7H\nwNgQDfqWNDZCye5zYX9gzcVh33sCRA205vwHx7hP4BtjXXz2C+qSNWu4K+WksqpaPt15hCVZhazf\nW0qjgX4xwcxyDN30iwm2u8Suz5jze/ZNwx7Ap4c1fBPW2wr78BTr6/AU6zn/TvwZ19fCicNQkQ8V\nBVCe73js+LqiAOprrK0fAyO0F5nDAAAM00lEQVQhKKrJ56hLfB0FPcLBq4VNaFxEw12pdjhWeZpP\ndhSxJKuIjQfKMAYG9Qo5G/QpUUF2l+gejIHSvXB8vzUT6OzHQetzbeX57QOjzg/8M6EfngI9E8Db\nx/n3rqm4OLTLzwR3PlQeAS7IveBY6y+OsCRr3n9gFNSUW/cpVJc6PpdYu4Sdrmj+fcXLWnvobOhH\nWp+b+wURmmD9MmgHDXelLlPxiRqWbreCfstBax760ISezEqPZ2ZaHEkRuuFHuxhjzes/vv9c2B8/\nAOWOx+X559btAWtbxdDEi0Mfc35on3l8Yfh6+1nHhyZaF4NDEx0hnnTu+bbsyVtfawV+dckF4d/c\ncyVQXcZFv0wmPATX/2/bf3ZouCvlUoXlp1i6vYjFWUVsyy8HYHhSGLPS45iRFkd8mC4j4DIN9daw\nyYWhf6bnX11yfvuA0OZDO8zxXFBMpwyXXFJjg/XL7GzvvwQiUiFuWLtOp+GuVAfJL6vmo+1FLMkq\nZMfhE4DVo79ucCzXDY7VWTcd7XSlFfLiZYV3QE+7K+pUGu5KdYL9JVV8suMIn2UXs+XQcYyBuNAA\nrh0Uw3VDYhnfJ9L5TcCVcoKGu1KdrPTkaVbuPsaKXcV8mXuM6toGAv28+Vr/KKYMjuXaQTFEBbdh\nbFepZmi4K2Wj0/UNbNhXxopdxXyWXUxhRQ0iMCIpjCmDY5k6JJb+McE6fKPaTMNdqS7CGMOuohN8\nln2UFdnFZBVYszmSInqcHacfmxqBr7eNF/2U29BwV6qLKj5Rw2fZR/ksu5g1eSWcrm8kJMCHqwdE\nM3VILJMHxBAa6Gt3maqL0nBXyg1U19azNq/UGr7JOUrJydN4ewljUsK5bnAsUwbHkqo3TqkmNNyV\ncjONjYZtBeVnh29yjlh3cSZHBHJVvyiu6hfJhL5RRAR14E5NqsvTcFfKzeWXVfN5zlHW5JWwYW8p\nlafrARgS15OJ/aOY0DeSsakRBPq14dZ85fZcFu4ikgT8A4jFuof2ZWPMsxe0EeBZYAZQDXzHGJPR\n0nk13JVyXn1DI9sPV7A2r4S1eaVsOXic2oZGfL2FEcnhTHT07NMTw/TCrIdzZbjHAXHGmAwRCQG2\nADcZY3Y1aTMDeAgr3K8EnjXGXNnSeTXclWq/U7UNbD5Yxpq8EtbmlbCz8ATGQLC/D1emRjiGcaIY\nEKvTLT2Ns+He6t9zxpgioMjxuFJEsoEEYFeTZjcC/zDWb4oNIhImInGOY5VSLtbDz5uv9Y/ma/2j\nATheVcv6faWOnn0Jn+UcBSA6xJ8JfSPPhn2CroHTbbRpsE5EUoARwFcXvJQA5Df5usDxnIa7Up0g\nPMiPGWnWImYABcerWZdXytq9VtgvzCwEIDUqiAl9I5nYL4rxfSMJC9SLs57K6XAXkWBgAfCIMeZE\ne95MROYB8wCSk5PbcwqllBMSwwO5ZUwgt4xJwhjDnuKTrMkrYV1eCf/eepi3vjqECFwR35NxqZFc\n2SeSsSkROr/egzg1W0ZEfIElwKfGmKebef0lYJUxZr7j693A5JaGZXTMXSl71DU0klVQzprcUtbv\nKyHjUDm19Y2IWDNxxvWJ5MrUCMamRmjPvgty5QVVAd4Ayowxj1yizUzgB5y7oPqcMWZsS+fVcFeq\na6ipayAzv5yv9pWxYV8pGYeOc9oR9oN79eTKPhFnA1/D3n6uDPeJwGpgO9DoePonQDKAMeZFxy+A\nvwDTsKZC3m2MaTG5NdyV6ppO1zewLb+CDftK2bDPmnZ5ut76rz+oVwjj+kQyrk8EY1Mj9YYqG+hN\nTEoplzhd30BWQQUb9pby1f4yNh8so6bu/LA/M4wTqUsadzgNd6VUh6itt8bsN+xzhP2B45yqs/Y8\nHRgbcnYYZ2xqhK5f3wE03JVSnaK2vpHth8vZ4Bizbxr2faKDGNM7gtEp4YxJiaB3ZKDeVHWZNNyV\nUrawZuNU8NX+UrYcOM7mg8epOFUHQFSwP2NSwhmdEsGYlHCGxPXER5dLaBOX3aGqlFJt4evtxaje\n4YzqHQ5Yq13mHTvJpgPWEM6mA2V8vOMIAIF+3oxIDmN07wjGpEQwIjmMIH+NJVfQnrtSqtMVVZxi\n84HjbD5QxqYDx8k+Yq2N4+0lDInreXYYZ3TvcGJ6BthdbpeiwzJKKbdxoqaOrYfKHWFfRmZ++dkZ\nOb0jAxl9dtw+nL7R3XsxNB2WUUq5jZ4Bvlw9IJqrB1gLodXWN7KzsMLq3R8sY9XuoyzIKAAgPNCX\nUb0jGNU7nOFJYaQlhhKsQzkX0Z+IUqrL8fPxYkRyOCOSw/kefTDGsL+k6uyY/aYDZazILgbAS6B/\nTAjDk8IYnhzGsMQwBsQGd/sLtToso5RyS2VVtWwrKCfzUDmZ+eVsKyinvNqaldPD15u0xFAr8B0f\ncaEBHjGco8MySimPFhHkxzUDY7hmYAwAxhgOllaTmV9+9uP1tQeobbDG7qND/M8L+/TEUEICPHcV\nTA13pZRHEBFSooJIiQriphEJgLV0Qk5R5XmBv3xXsaM99IsOZliTwB/YK8RjtinUYRmlVLdSXl3L\ntoIKMg9ZQzmZ+eWUVdUCEODrxdD40LMXatMTw+gdEYiXV9cZztFhGaWUakZYoN95M3OMMeSXnSLz\n7Pj9cf6x4SC1jpUwQ/x9GJoQSnpi6NnPyRFdfxkFDXelVLcmIiRHBpIcGcicYfGAtYRCbvFJth8u\nZ/vhCrYXVPD3JuP3PQN8SEsMJS0hjDRH4CeG9+hSga/DMkop5YTa+kb2FFey/XAFWQUVbD9czu4j\nldQ1WBkaFuhLWkLo2bAfmhBKQpjrA1/vUFVKqQ52ur6B3UcqySqoYIcj9PcUV1LfaOVqRJDf2cC3\nxvBD6dXz8qZk6pi7Ukp1MH8fb9ITw0hPDDv7XE1dAzlHKtleUO7o4VewJq+EBkfgRwX7cd+kvnxv\nUp8OrU3DXSmlXCjA1/vs1MozTtU2sKvoxNnefUzPjt/ERMNdKaU6WA8/7/OWQe4MnjFbXyml1Hk0\n3JVSygNpuCullAfScFdKKQ+k4a6UUh5Iw10ppTyQhrtSSnkgDXellPJAtq0tIyLHgIPtPDwKKHFh\nOZ1Ja7eH1m4Pd629K9fd2xgT3Voj28L9cojIZmcWzumKtHZ7aO32cNfa3bXupnRYRimlPJCGu1JK\neSB3DfeX7S7gMmjt9tDa7eGutbtr3We55Zi7Ukqplrlrz10ppVQL3C7cRWSaiOwWkTwRecLuepwl\nIkkislJEdonIThF52O6a2kJEvEVkq4gssbuWthCRMBH5QERyRCRbRMbbXZOzROQ/Hf9WdojIfBEJ\nsLumSxGR10TkqIjsaPJchIgsF5Fcx+fOW8y8DS5R+x8c/2ayRORfIhLW0jm6IrcKdxHxBv4KTAeG\nALeLyBB7q3JaPfCoMWYIMA540I1qB3gYyLa7iHZ4FvjEGDMIGIabfA8ikgD8EBhtjBkKeAO32VtV\ni14Hpl3w3BPAZ8aY/sBnjq+7ote5uPblwFBjTDqwB3iys4u6XG4V7sBYIM8Ys88YUwu8A9xoc01O\nMcYUGWMyHI8rsUImwd6qnCMiicBM4BW7a2kLEQkFJgGvAhhjao0x5fZW1SY+QA8R8QECgUKb67kk\nY8yXQNkFT98IvOF4/AZwU6cW5aTmajfGLDPG1Du+3AAkdnphl8ndwj0ByG/ydQFuEpBNiUgKMAL4\nyt5KnPYM8DjQaHchbZQKHAP+7hhSekVEguwuyhnGmMPAH4FDQBFQYYxZZm9VbRZrjClyPD4CxNpZ\nzGW4B/jY7iLayt3C3e2JSDCwAHjEGHPC7npaIyKzgKPGmC1219IOPsBI4AVjzAigiq47NHAex/j0\njVi/oOKBIBGZa29V7WesaXluNzVPRP4ba0j1LbtraSt3C/fDQFKTrxMdz7kFEfHFCva3jDEf2l2P\nk64C5ojIAaxhsGtF5E17S3JaAVBgjDnzF9IHWGHvDq4D9htjjhlj6oAPgQk219RWxSISB+D4fNTm\netpERL4DzALuNG44Z9zdwn0T0F9EUkXED+sC0yKba3KKiAjW2G+2MeZpu+txljHmSWNMojEmBevn\n/bkxxi16kMaYI0C+iAx0PDUF2GVjSW1xCBgnIoGOfztTcJOLwU0sAu5yPL4LWGhjLW0iItOwhiLn\nGGOq7a6nPdwq3B0XOH4AfIr1D/09Y8xOe6ty2lXAt7B6vpmOjxl2F9UNPAS8JSJZwHDgtzbX4xTH\nXxsfABnAdqz/q132rkkRmQ+sBwaKSIGI3As8BUwVkVysv0SesrPGS7lE7X8BQoDljv+rL9paZDvo\nHapKKeWB3KrnrpRSyjka7kop5YE03JVSygNpuCullAfScFdKKQ+k4a6UUh5Iw10ppTyQhrtSSnmg\n/w/9HCAS5s3dKQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tXq5HncUiIP5",
        "colab_type": "text"
      },
      "source": [
        "From the plot, we can infer that validation loss has increased after epoch 17 for 2 successive epochs. Hence, training is stopped at epoch 19.\n",
        "\n",
        "Next, let’s build the dictionary to convert the index to word for target and source vocabulary:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6AjbYmCjiF2N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "reverse_target_word_index=y_tokenizer.index_word\n",
        "reverse_source_word_index=x_tokenizer.index_word\n",
        "target_word_index=y_tokenizer.word_index"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m9x1ntd6iMuM",
        "colab_type": "text"
      },
      "source": [
        "## Inference\n",
        "Set up the inference for the encoder and decoder:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yOJEqZX9iKpj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Encode the input sequence to get the feature vector\n",
        "encoder_model = Model(inputs=encoder_inputs,outputs=[encoder_outputs, state_h, state_c])\n",
        "\n",
        "# Decoder setup\n",
        "# Below tensors will hold the states of the previous time step\n",
        "decoder_state_input_h = Input(shape=(latent_dim,))\n",
        "decoder_state_input_c = Input(shape=(latent_dim,))\n",
        "decoder_hidden_state_input = Input(shape=(max_text_len,latent_dim))\n",
        "\n",
        "# Get the embeddings of the decoder sequence\n",
        "dec_emb2= dec_emb_layer(decoder_inputs) \n",
        "# To predict the next word in the sequence, set the initial states to the states from the previous time step\n",
        "decoder_outputs2, state_h2, state_c2 = decoder_lstm(dec_emb2, initial_state=[decoder_state_input_h, decoder_state_input_c])\n",
        "\n",
        "#attention inference\n",
        "attn_out_inf, attn_states_inf = attn_layer([decoder_hidden_state_input, decoder_outputs2])\n",
        "decoder_inf_concat = Concatenate(axis=-1, name='concat')([decoder_outputs2, attn_out_inf])\n",
        "\n",
        "# A dense softmax layer to generate prob dist. over the target vocabulary\n",
        "decoder_outputs2 = decoder_dense(decoder_inf_concat) \n",
        "\n",
        "# Final decoder model\n",
        "decoder_model = Model(\n",
        "    [decoder_inputs] + [decoder_hidden_state_input,decoder_state_input_h, decoder_state_input_c],\n",
        "    [decoder_outputs2] + [state_h2, state_c2])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NszXPeuViWZF",
        "colab_type": "text"
      },
      "source": [
        "We are defining a function below which is the implementation of the inference process (which we covered here):\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UfmexOrLiU0E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def decode_sequence(input_seq):\n",
        "    # Encode the input as state vectors.\n",
        "    e_out, e_h, e_c = encoder_model.predict(input_seq)\n",
        "    \n",
        "    # Generate empty target sequence of length 1.\n",
        "    target_seq = np.zeros((1,1))\n",
        "    \n",
        "    # Populate the first word of target sequence with the start word.\n",
        "    target_seq[0, 0] = target_word_index['sostok']\n",
        "\n",
        "    stop_condition = False\n",
        "    decoded_sentence = ''\n",
        "    while not stop_condition:\n",
        "      \n",
        "        output_tokens, h, c = decoder_model.predict([target_seq] + [e_out, e_h, e_c])\n",
        "\n",
        "        # Sample a token\n",
        "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
        "        sampled_token = reverse_target_word_index[sampled_token_index]\n",
        "        \n",
        "        if(sampled_token!='eostok'):\n",
        "            decoded_sentence += ' '+sampled_token\n",
        "\n",
        "        # Exit condition: either hit max length or find stop word.\n",
        "        if (sampled_token == 'eostok'  or len(decoded_sentence.split()) >= (max_summary_len-1)):\n",
        "            stop_condition = True\n",
        "\n",
        "        # Update the target sequence (of length 1).\n",
        "        target_seq = np.zeros((1,1))\n",
        "        target_seq[0, 0] = sampled_token_index\n",
        "\n",
        "        # Update internal states\n",
        "        e_h, e_c = h, c\n",
        "\n",
        "    return decoded_sentence"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ydEajpIOieU7",
        "colab_type": "text"
      },
      "source": [
        "Let us define the functions to convert an integer sequence to a word sequence for summary as well as the reviews:\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cJ_ydDC9ichh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def seq2summary(input_seq):\n",
        "    newString=''\n",
        "    for i in input_seq:\n",
        "        if((i!=0 and i!=target_word_index['sostok']) and i!=target_word_index['eostok']):\n",
        "            newString=newString+reverse_target_word_index[i]+' '\n",
        "    return newString\n",
        "\n",
        "def seq2text(input_seq):\n",
        "    newString=''\n",
        "    for i in input_seq:\n",
        "        if(i!=0):\n",
        "            newString=newString+reverse_source_word_index[i]+' '\n",
        "    return newString"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p5uexyXOiikl",
        "colab_type": "text"
      },
      "source": [
        "Here are a few summaries generated by the model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zIRZSuzWigbG",
        "colab_type": "code",
        "outputId": "d4d82720-6a1f-4e45-c9c5-35184bf8df21",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 8537
        }
      },
      "source": [
        "for i in range(0,100):\n",
        "    print(\"Review:\",seq2text(x_tr[i]))\n",
        "    print(\"Original summary:\",seq2summary(y_tr[i]))\n",
        "    print(\"Predicted summary:\",decode_sequence(x_tr[i].reshape(1,max_text_len)))\n",
        "    print(\"\\n\")"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Review: gave caffeine shakes heart anxiety attack plus tastes unbelievably bad stick coffee tea soda thanks \n",
            "Original summary: hour \n",
            "Predicted summary:  not what expected\n",
            "\n",
            "\n",
            "Review: got great course good belgian chocolates better \n",
            "Original summary: would like to give it stars but \n",
            "Predicted summary:  yummy\n",
            "\n",
            "\n",
            "Review: one best flavored coffees tried usually like flavored coffees one great serve company love \n",
            "Original summary: delicious \n",
            "Predicted summary:  great coffee\n",
            "\n",
            "\n",
            "Review: salt separate area pain makes hard regulate salt putting like salt go ahead get product \n",
            "Original summary: tastes ok packaging \n",
            "Predicted summary:  salt\n",
            "\n",
            "\n",
            "Review: really like product super easy order online delivered much cheaper buying gas station stocking good long drives \n",
            "Original summary: turkey jerky is great \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: best salad dressing delivered promptly quantities last vidalia onion dressing compares made oak hill farms sometimes find costco order front door want even orders cut shipping costs \n",
            "Original summary: my favorite salad dressing \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: think sitting around warehouse long time took long time send got tea tasted like cardboard red rasberry leaf tea know supposed taste like \n",
            "Original summary: stale \n",
            "Predicted summary:  not the same as expected\n",
            "\n",
            "\n",
            "Review: year old cat special diet digestive problems also diabetes stopped eating usual special formula food tried different kinds catfood one liked easy digestion diabetes thank newman \n",
            "Original summary: wonderful \n",
            "Predicted summary:  my cat loves this food\n",
            "\n",
            "\n",
            "Review: always perfect snack dog loves knows exactly starts ask time evening gets greenie snack thank excellent product fast delivery \n",
            "Original summary: greenies buddy treat \n",
            "Predicted summary:  great treat\n",
            "\n",
            "\n",
            "Review: dog loves tiny treats keep one car one house \n",
            "Original summary: dog loves them \n",
            "Predicted summary:  dog loves them\n",
            "\n",
            "\n",
            "Review: liked coffee much subscribing dark rich smooth \n",
            "Original summary: makes great cup of java \n",
            "Predicted summary:  great coffee\n",
            "\n",
            "\n",
            "Review: far dog tried chicken peanut butter flavor absolutely loves love natural makes happy giving dog something healthy treats small soft big plus calories \n",
            "Original summary: love zuke mini naturals \n",
            "Predicted summary:  my dog loves these\n",
            "\n",
            "\n",
            "Review: absolutely delicious satisfy something sweet really filling great early morning time make breakfast great afternoon snack work feeling sluggish \n",
            "Original summary: protein bar \n",
            "Predicted summary:  delicious\n",
            "\n",
            "\n",
            "Review: aware decaf coffee although showed search decaf cups intended purchase gift kept recipient drink caffeine favorite means \n",
            "Original summary: not decaf \n",
            "Predicted summary:  not bad\n",
            "\n",
            "\n",
            "Review: wonderful wrote perfect iced cookie one pen writing cookies names happy ca \n",
            "Original summary: cookie \n",
            "Predicted summary:  great cookies\n",
            "\n",
            "\n",
            "Review: truffle oil quite good prefer brand france urbani italy expensive oh delicious tried black white good black bit stronger pungent event healthy alternative butter enjoy \n",
            "Original summary: delicious but not the best \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: enjoy coffee office split right middle loving think worth try order regularly \n",
            "Original summary: hit or miss \n",
            "Predicted summary:  great coffee\n",
            "\n",
            "\n",
            "Review: husband gluten free food several years tried several different bread mixes first actually enjoys buying amazon saves loaf \n",
            "Original summary: really good gluten free bread \n",
            "Predicted summary:  pamela gf bread mix\n",
            "\n",
            "\n",
            "Review: hubby eats says good snacks morning done apple flavor \n",
            "Original summary: really good nice snack \n",
            "Predicted summary:  great snack\n",
            "\n",
            "\n",
            "Review: waste money disgusting product chocolate taste tastes like plastic lining paper carton using milk treated ultra high temperatures like fresh milk go get fresh milk hershey syrup want chocolate milk \n",
            "Original summary: please do not waste your money \n",
            "Predicted summary:  not bad\n",
            "\n",
            "\n",
            "Review: absolutely loves apple chicken happy hips looks forward one morning one night gets soooo excited would eat allowed \n",
            "Original summary: healthy treats \n",
            "Predicted summary:  my loves it\n",
            "\n",
            "\n",
            "Review: strong much flavor little aroma tried purchase another time similiar brands met standards expected \n",
            "Original summary: no flavor \n",
            "Predicted summary:  good coffee\n",
            "\n",
            "\n",
            "Review: company wanted chose order anyway \n",
            "Original summary: water \n",
            "Predicted summary:  good product\n",
            "\n",
            "\n",
            "Review: introduced number people hooked best sour gummy ever great flavors got great price \n",
            "Original summary: new favorite \n",
            "Predicted summary:  great\n",
            "\n",
            "\n",
            "Review: new price attractive however tastes horrible maybe old zico coconut water brands might find acceptable \n",
            "Original summary: do not be by the price \n",
            "Predicted summary:  horrible\n",
            "\n",
            "\n",
            "Review: sure ever going buy product way expensive market price \n",
            "Original summary: too expensive \n",
            "Predicted summary:  good stuff\n",
            "\n",
            "\n",
            "Review: flavor normally find local stores plus buy bulk things take savings add veggies even stir egg noodles cook add nutrition quick meals lot extra \n",
            "Original summary: good value \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: order tea labeled decaff must caffeine residue levels tested tea caffeine decaff non decaff tea anywhere caffeine caffeine caffeinated tea caffeine slightly less naturally present tea leaf \n",
            "Original summary: caffeine is not \n",
            "Predicted summary:  not as good as expected\n",
            "\n",
            "\n",
            "Review: excellent babies toddler really best offer little one delicious rich vitamins calcium protein low fat sorry products available website \n",
            "Original summary: excellent product for babies and toddler \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: purchased item dented would bet run dented product clearing ship ones \n",
            "Original summary: sometimes dented \n",
            "Predicted summary:  dented cans\n",
            "\n",
            "\n",
            "Review: almost tastes like mini blueberry pie love one favorite thoroughly fallen love \n",
            "Original summary: excellent love the blueberry pecan \n",
            "Predicted summary:  delicious\n",
            "\n",
            "\n",
            "Review: dog loves keeps busy minutes long time chew hound \n",
            "Original summary: chew away \n",
            "Predicted summary:  dog loves it\n",
            "\n",
            "\n",
            "Review: plant came quickly looks great office nice pot plant thriving well \n",
            "Original summary: very nice office plant \n",
            "Predicted summary:  great gift\n",
            "\n",
            "\n",
            "Review: dog loves lickety stik bacon flavor since likes much plan getting flavors great liquid treat dog highly recommend lickety stik \n",
            "Original summary: great dog treat \n",
            "Predicted summary:  dog loves them\n",
            "\n",
            "\n",
            "Review: great toy dogs chew everything else little literally eats toys one toys yet destroy loves carries around everywhere got rex cutest thing \n",
            "Original summary: good for chewers \n",
            "Predicted summary:  dogs love it\n",
            "\n",
            "\n",
            "Review: really search good deals tea tea great price tea amazon almost cup price cup coffee herbal varieties low caffine good option wife used dinner coffe \n",
            "Original summary: great price for great tea \n",
            "Predicted summary:  great tea\n",
            "\n",
            "\n",
            "Review: pricey essentially small bag hard crumbs maybe dog spoiled treats like third class treats definitely bottom doggie treat often simply walk away glad people like buying \n",
            "Original summary: waste of money \n",
            "Predicted summary:  my dog loves these\n",
            "\n",
            "\n",
            "Review: little pricey consider sugar low cal caffine really rich flavor best chai ever found \n",
            "Original summary: fabulous product \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: loves taste beef freeze dried dog treats use training really works \n",
            "Original summary: dog lover \n",
            "Predicted summary:  my dog loves these\n",
            "\n",
            "\n",
            "Review: three dogs cairn terriers year old border collie proud greenies like taste helps keep gums teeth good shape \n",
            "Original summary: our dogs love greenies \n",
            "Predicted summary:  dogs love them\n",
            "\n",
            "\n",
            "Review: good soft drink smooth strawberry cream soda tasty \n",
            "Original summary: good stuff \n",
            "Predicted summary:  refreshing\n",
            "\n",
            "\n",
            "Review: item arrived sugar free shipped regular version caramel syrup small internal sticker bottle stated sugar free although company label bottle stated regular version \n",
            "Original summary: wrong item \n",
            "Predicted summary:  not what expected\n",
            "\n",
            "\n",
            "Review: like strong coffee coffee rated found weak sickening taste \n",
            "Original summary: disapointed \n",
            "Predicted summary:  not bad\n",
            "\n",
            "\n",
            "Review: saw peanut butter chocolate cereal knew try pleased eat chocolate breakfast feel guilty two kids love cereal well great eat alone favorite milk product yogurt mix homemade granola well \n",
            "Original summary: the yummy \n",
            "Predicted summary:  delicious\n",
            "\n",
            "\n",
            "Review: begging time loves used buy small bottle buying every weeks since saw oz buying last lot longer gas money cheaper buy online \n",
            "Original summary: my dog loves it \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: true also need decent scale tried caviar recipe everything worked perfectly first try fun easy make kit comes large enough samples looks like good uses \n",
            "Original summary: great to \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: dog really likes treats like buy run mill treats loaded fat fillers continue buy \n",
            "Original summary: buddy biscuits \n",
            "Predicted summary:  my dog loves these\n",
            "\n",
            "\n",
            "Review: tulsi green tea great good iced tea well \n",
            "Original summary: green tea \n",
            "Predicted summary:  tea\n",
            "\n",
            "\n",
            "Review: always put something market couple poof gone best tasting product pepsi \n",
            "Original summary: best taste \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: like tomatoes fresh flavorful also come carton welcome alternative metal cans impart flavor sometimes lined plastic containing \n",
            "Original summary: yummy tomatoes good packaging \n",
            "Predicted summary:  not what expected\n",
            "\n",
            "\n",
            "Review: great get habit forming careful bought whole case save overall versus going supermarket rich dark chocolate crisp cookie worth every penny oreo eat heart \n",
            "Original summary: delicious \n",
            "Predicted summary:  great tasting\n",
            "\n",
            "\n",
            "Review: else say arrived promptly perhaps time expected expiration date like next day good go \n",
            "Original summary: baby loves it \n",
            "Predicted summary:  not what expected\n",
            "\n",
            "\n",
            "Review: bought local recently advertised cheesy flavor detectable product even salt flavor avoid product \n",
            "Original summary: no cheese flavor \n",
            "Predicted summary:  not bad\n",
            "\n",
            "\n",
            "Review: big volume coffee morning one great \n",
            "Original summary: great morning coffee \n",
            "Predicted summary:  great coffee\n",
            "\n",
            "\n",
            "Review: drank try keep awake fell asleep minutes drinking feel anything \n",
            "Original summary: it made me fall \n",
            "Predicted summary:  great coffee\n",
            "\n",
            "\n",
            "Review: drink cups day verona italian french roast coffee wanted try lower acid version brand coffee smells tastes like vinegar totally unpalatable better drinking water acid coffee bothers \n",
            "Original summary: single worst coffee ever \n",
            "Predicted summary:  good coffee\n",
            "\n",
            "\n",
            "Review: getting price however afraid stocking anymore reduced price think one trying eat crackers low calorie string cheese breakfast every total calories put breakfast baggie go \n",
            "Original summary: am addicted to these \n",
            "Predicted summary:  great for diabetics\n",
            "\n",
            "\n",
            "Review: first time using fondarific fondant general one really easy use baby shower cake worked indicated also colored made two tier cake final product looked great greasy \n",
            "Original summary: easy to use \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: work home drink cups cup coffee day good tasting coffee lowest price cup market \n",
            "Original summary: great coffee great price \n",
            "Predicted summary:  great coffee\n",
            "\n",
            "\n",
            "Review: guys say natural really tastes great pleasantly surprised stand flavor carbonated think would even better product time come fed sweet juices aftertaste make obvious really natural switch really gets vote \n",
            "Original summary: great taste all natural \n",
            "Predicted summary:  refreshing\n",
            "\n",
            "\n",
            "Review: product good goes long way quite good one dd good product less \n",
            "Original summary: very good \n",
            "Predicted summary:  good stuff\n",
            "\n",
            "\n",
            "Review: tea wonderful soothing even soothing get shipped house found hard find decaffeinated tea grocery store much easier \n",
            "Original summary: decaffeinated french vanilla tea yummy \n",
            "Predicted summary:  great tea\n",
            "\n",
            "\n",
            "Review: wow little calorie espresso sugar serve cold delicious little shot espresso sugar overly sweet sugar helps offset taste espresso caffe bitter sweet tastes good really gave afternoon kick pants \n",
            "Original summary: nice little pick me up \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: mayonnaise delicious side side taste test would give hellman edge hellman richer taste \n",
            "Original summary: excellent but \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: love medium full flavored roast smooth taste bitter acidic taste excellent coffee good value also try timothy kona good also \n",
            "Original summary: wonderful coffee \n",
            "Predicted summary:  great coffee\n",
            "\n",
            "\n",
            "Review: nice item chunks meat good gravy cat fond varieties nice little treat nonetheless think item bit pricy per ounce \n",
            "Original summary: nice but pricey \n",
            "Predicted summary:  cat food\n",
            "\n",
            "\n",
            "Review: bought cookies gifts open last long good make great gifts would definitely buy \n",
            "Original summary: mouth watery cookies \n",
            "Predicted summary:  great cookies\n",
            "\n",
            "\n",
            "Review: great price fast shipping best chips better ingredients less calories snack foods plus taste like real chips \n",
            "Original summary: pop chips are the best \n",
            "Predicted summary:  great chips\n",
            "\n",
            "\n",
            "Review: taco bell chipotle sauce bold flavorful tried chicken wings tacos salad made dish extremely tasty glad sampled new sauce staple condiment \n",
            "Original summary: bold flavor \n",
            "Predicted summary:  great cocoa\n",
            "\n",
            "\n",
            "Review: bought seeds make centerpieces really surprised fast grow planted seeds potting soil without ny preparation anything kept watering days super tall ready displayed centerpieces perfect \n",
            "Original summary: perfect for in days \n",
            "Predicted summary:  great gift\n",
            "\n",
            "\n",
            "Review: every time need sun dried tomatoes local grocery stores conveniently small pouches ensure always hand called recipe \n",
            "Original summary: sun dried tomato bliss \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: love soup eat plain use recipe cannot find area glad amazon \n",
            "Original summary: soup chicken cheese \n",
            "Predicted summary:  great soup\n",
            "\n",
            "\n",
            "Review: size quite good dog training smell strong cannot put open bag must seal everytime gave treat otherwise dog stand trying fetch believe taste great puppy purchase sure \n",
            "Original summary: strong smell and my puppy loves it \n",
            "Predicted summary:  my dog loves these\n",
            "\n",
            "\n",
            "Review: love chips auto order every months taste great whole bag calories bag every day sure helped weight loss little bags eat huge amount \n",
            "Original summary: great purchase \n",
            "Predicted summary:  great chips\n",
            "\n",
            "\n",
            "Review: many kit wines cost three four times made many kits find fine table wine recommend adding water five gallon mark flavor \n",
            "Original summary: good wine \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: sooo much pepper heavy salt reminds adams trick food cannot eat seriously fresh nuts seasoned \n",
            "Original summary: over the top seasoning \n",
            "Predicted summary:  great soup\n",
            "\n",
            "\n",
            "Review: loved brand best vanilla flavor others tried would buy better price \n",
            "Original summary: wolfgang puck coffee vanilla \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: another brand cinammon carried amazon much better tasting brand maybe packaging part problem simple plastic bag tie amazon brand comes carefully set plastic box \n",
            "Original summary: edible have had much better \n",
            "Predicted summary:  not as good as expected\n",
            "\n",
            "\n",
            "Review: throw pack one actually taste bad especially compared orange tangerine like carbonation adds juice flavors need work switch drinks best worst watermelon strawberry kiwi berry black cherry orange tangerine \n",
            "Original summary: my favorite of the four tried \n",
            "Predicted summary:  not bad\n",
            "\n",
            "\n",
            "Review: daughter drinking since months old months old still loves snack time healthy delicious great addition menu \n",
            "Original summary: great snack \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: live guinea africa order products delivered boat every months sometimes disappointed time zero calories zero carbs taste great price zero delivery costs prime ordered different flavors one favorite love \n",
            "Original summary: love it \n",
            "Predicted summary:  good but not the best\n",
            "\n",
            "\n",
            "Review: purchased larger size love size perfect keep purse snack especially times others dessert snack cannot eat must gluten free spouse touch diet food loves \n",
            "Original summary: cannot get enough \n",
            "Predicted summary:  great snack\n",
            "\n",
            "\n",
            "Review: always house drink favorite mix sprite oh good every day mind larger bottles use much bring \n",
            "Original summary: am an adult still love this \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: ginger snaps overpowering ginger go great milk really enjoyed house great buy affordable compared alternative diet foods last least week store well \n",
            "Original summary: you can eat ginger again \n",
            "Predicted summary:  great tasting\n",
            "\n",
            "\n",
            "Review: give squid one star use might thoroughly disappointed quite possibly call crazy \n",
            "Original summary: can for your \n",
            "Predicted summary:  not what expected\n",
            "\n",
            "\n",
            "Review: quality seeds excellent begin germinate hours days ready use never sprouted seeds results good easily recommend sprouter whether human consumption four legged friends \n",
            "Original summary: wheat grass seeds \n",
            "Predicted summary:  great gift\n",
            "\n",
            "\n",
            "Review: love stuff great store bought homemade baked goods kicking things professional level works colored dark light frosting also used dusting powdered sugar pretty fine texture \n",
            "Original summary: fun like dust \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: bought jumbo greenies black lab loved way expensive regular use notice difference breath primary reason buying \n",
            "Original summary: jumbo greenies good but very expensive \n",
            "Predicted summary:  greenies\n",
            "\n",
            "\n",
            "Review: also bought costco per box included bags oz kids fighting remaining bags good buying due price high price prevent product reaching mass distribution \n",
            "Original summary: very good but too pricey \n",
            "Predicted summary:  good product\n",
            "\n",
            "\n",
            "Review: originally found mints whole foods taste superb get lot money plus comes cute little tin uses dog loves go organic \n",
            "Original summary: wonderful \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: regular spam awful almost inedible would give tastes like animal know mean fellow spam turkey spam pretty good great would give worth try \n",
            "Original summary: better than regular \n",
            "Predicted summary:  not bad\n",
            "\n",
            "\n",
            "Review: really need know many cans also whitefish tuna buffet canned cat food thanks \n",
            "Original summary: need to know how many in case \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: great tasting rich flavor perfect making nice hot cup mocha bought test hershey syrup mocha incredible distinct taste difference noticeable much richer tastes like chocolate less sugary hershey syrup \n",
            "Original summary: great taste \n",
            "Predicted summary:  best ever\n",
            "\n",
            "\n",
            "Review: number one japan number one great save get shipped automatically every month lugging car \n",
            "Original summary: great tea \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: bought item read best mayo sold yes even better worlds favorite hellman well review good bit better hellman fact put empty hellman jar said nothing family never knew difference \n",
            "Original summary: blue mayo \n",
            "Predicted summary:  good product\n",
            "\n",
            "\n",
            "Review: gum great makes car smell good leave refreshing sweet tart smooth \n",
            "Original summary: love the gum and the price \n",
            "Predicted summary:  refreshing\n",
            "\n",
            "\n",
            "Review: flavorful smells like heaven great price compared stores arrived fast \n",
            "Original summary: divine \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: love low calorie organic doctors recommend grams fiber daily smart bran grams per serving fruits veggies set day eat dry vanilla frozen yogurt cinnamon \n",
            "Original summary: yes to smart bran \n",
            "Predicted summary:  great product\n",
            "\n",
            "\n",
            "Review: found spice blend dallas years back tell restaurant using grilled shrimp like cajun spice grilling fish recommend store dry place replace every year least lose flavor \n",
            "Original summary: good stuff \n",
            "Predicted summary:  best seasoning ever\n",
            "\n",
            "\n",
            "Review: plain riceselect couscous delicious easy quick prepare great side item base main course far found bad product riceselect \n",
            "Original summary: yummy \n",
            "Predicted summary:  good but not great\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e_t0XmysikdD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}