material/deep_learning/next_word_prediction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.1"
    },
    "colab": {
      "name": "next_word_prediction.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "Pyz8OyCRKRz8",
        "rFwbg7k1KZrK",
        "OOH1dbiHh5_S",
        "bobHiFxmBN01",
        "2HfIzDjeY0d7",
        "5Txqhf6XYFyP",
        "s5JdxQn-bNPp",
        "PvnNncG_3p87"
      ],
      "toc_visible": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pyz8OyCRKRz8"
      },
      "source": [
        "# Bert를 사용한 다음 단어 예측\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rFwbg7k1KZrK"
      },
      "source": [
        "# 필요 라이브러리 설치"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GTUt2OoT04Kh",
        "outputId": "7a8dbe06-2ce9-4e5b-b8ae-c5cecfbca291"
      },
      "source": [
        "!pip install transformers==3.0.2\n",
        "!pip install sentencepiece"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers==3.0.2\n",
            "  Downloading transformers-3.0.2-py3-none-any.whl (769 kB)\n",
            "\u001b[K     |████████████████████████████████| 769 kB 16.1 MB/s \n",
            "\u001b[?25hCollecting tokenizers==0.8.1.rc1\n",
            "  Downloading tokenizers-0.8.1rc1-cp37-cp37m-manylinux1_x86_64.whl (3.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0 MB 38.7 MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "  Downloading sacremoses-0.0.53.tar.gz (880 kB)\n",
            "\u001b[K     |████████████████████████████████| 880 kB 35.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==3.0.2) (3.7.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from transformers==3.0.2) (1.21.6)\n",
            "Collecting sentencepiece!=0.1.92\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 52.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==3.0.2) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==3.0.2) (4.64.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==3.0.2) (2022.6.2)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==3.0.2) (21.3)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==3.0.2) (3.0.9)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.0.2) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.0.2) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.0.2) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.0.2) (2022.6.15)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3.0.2) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3.0.2) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3.0.2) (1.1.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895260 sha256=ed8ebe715d8eb27185aa1b88b19852f6ab9ed21538af97456b483a2875e3dbd3\n",
            "  Stored in directory: /root/.cache/pip/wheels/87/39/dd/a83eeef36d0bf98e7a4d1933a4ad2d660295a40613079bafc9\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sentencepiece, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.53 sentencepiece-0.1.96 tokenizers-0.8.1rc1 transformers-3.0.2\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (0.1.96)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kVofHG0eh40t",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b49ccb28-7eb4-44f6-a7ba-f7309f62f6f5"
      },
      "source": [
        "!pip install konlpy"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting konlpy\n",
            "  Downloading konlpy-0.6.0-py2.py3-none-any.whl (19.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 19.4 MB 34.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.6 in /usr/local/lib/python3.7/dist-packages (from konlpy) (1.21.6)\n",
            "Collecting JPype1>=0.7.0\n",
            "  Downloading JPype1-1.4.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (453 kB)\n",
            "\u001b[K     |████████████████████████████████| 453 kB 65.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: lxml>=4.1.0 in /usr/local/lib/python3.7/dist-packages (from konlpy) (4.2.6)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from JPype1>=0.7.0->konlpy) (4.1.1)\n",
            "Installing collected packages: JPype1, konlpy\n",
            "Successfully installed JPype1-1.4.0 konlpy-0.6.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OOH1dbiHh5_S"
      },
      "source": [
        "# 셋업"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C-FHX9tOK4pV"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from tqdm import tqdm\n",
        "\n",
        "from transformers import BertTokenizer\n",
        "from transformers import TFBertModel\n",
        "\n",
        "import tensorflow as tf"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ex8GjpDqK7Lm"
      },
      "source": [
        "#random seed 고정\n",
        "tf.random.set_seed(1234)\n",
        "np.random.seed(1234)\n",
        "\n",
        "CUSTOM_VOCAB_FILE = 'custom_vocab.txt'\n",
        "MAX_VOCAB_SIZE = 10000*2\n",
        "\n",
        "BERT_MODEL_NAME = 'bert-base-multilingual-cased'\n",
        "BERT_SEQUENCE_LENGTH = 128"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "104nTDhQLQmP"
      },
      "source": [
        "# 데이터"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bobHiFxmBN01"
      },
      "source": [
        "## 데이터 다운로드"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2KhJDoXqJyQP",
        "outputId": "914dbcb2-9932-43e2-b9a1-cded632ba779"
      },
      "source": [
        "!wget https://raw.githubusercontent.com/dhrim/keras_howto_2021/master/data/alice_in_wonderland.txt"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-07-11 04:24:14--  https://raw.githubusercontent.com/dhrim/keras_howto_2021/master/data/alice_in_wonderland.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.108.133, 185.199.109.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 172864 (169K) [text/plain]\n",
            "Saving to: ‘alice_in_wonderland.txt’\n",
            "\n",
            "\ralice_in_wonderland   0%[                    ]       0  --.-KB/s               \ralice_in_wonderland 100%[===================>] 168.81K  --.-KB/s    in 0.008s  \n",
            "\n",
            "2022-07-11 04:24:14 (21.1 MB/s) - ‘alice_in_wonderland.txt’ saved [172864/172864]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4e_WDU3-J1PM",
        "outputId": "6eddb18d-9239-4faa-9816-efc98f63defe"
      },
      "source": [
        "!head -n 20 alice_in_wonderland.txt"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Alice’s Adventures in Wonderland\r\n",
            "\r\n",
            "by Lewis Carroll\r\n",
            "\r\n",
            "CHAPTER I.\r\n",
            "Down the Rabbit-Hole\r\n",
            "\r\n",
            "\r\n",
            "Alice was beginning to get very tired of sitting by her sister on the\r\n",
            "bank, and of having nothing to do: once or twice she had peeped into\r\n",
            "the book her sister was reading, but it had no pictures or\r\n",
            "conversations in it, “and what is the use of a book,” thought Alice\r\n",
            "“without pictures or conversations?”\r\n",
            "\r\n",
            "So she was considering in her own mind (as well as she could, for the\r\n",
            "hot day made her feel very sleepy and stupid), whether the pleasure of\r\n",
            "making a daisy-chain would be worth the trouble of getting up and\r\n",
            "picking the daisies, when suddenly a White Rabbit with pink eyes ran\r\n",
            "close by her.\r\n",
            "\r\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0DK2dQ5mJ2-t",
        "outputId": "7099b624-9d0f-43f0-ac6e-69c22b8ed1ff"
      },
      "source": [
        "!wc alice_in_wonderland.txt"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  3710  29378 172864 alice_in_wonderland.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QZu1lTyM28Rf"
      },
      "source": [
        "## 데이터 로딩"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h4K1hOXQJ4qG"
      },
      "source": [
        "def load_raw_text_from_file(file_path):\n",
        "  with open(file_path, 'r', encoding='utf-8') as fp: raw_text = fp.read()\n",
        "  return raw_text"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Aa966BcXJ7ib",
        "outputId": "a65055bc-1a76-444f-9836-a85b2e4f5743"
      },
      "source": [
        "import collections\n",
        "from collections import OrderedDict\n",
        "\n",
        "# 파일을 읽고\n",
        "raw_text = load_raw_text_from_file('alice_in_wonderland.txt')\n",
        "\n",
        "print(\"len(raw_text)=\", len(raw_text))\n",
        "\n",
        "print(raw_text[:100])"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "len(raw_text)= 162663\n",
            "Alice’s Adventures in Wonderland\n",
            "\n",
            "by Lewis Carroll\n",
            "\n",
            "CHAPTER I.\n",
            "Down the Rabbit-Hole\n",
            "\n",
            "\n",
            "Alice was begi\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wjcoCHX5uHnE"
      },
      "source": [
        "## token으로 짜르기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "owsW1JvOK0tn",
        "outputId": "afa30df8-a2e7-403e-9eb9-fb59d222c68d"
      },
      "source": [
        "raw_text = raw_text.replace(\"\\n\", \" \")\n",
        "raw_text = raw_text.replace(\"  \", \" \")\n",
        "raw_text = raw_text.replace(\"“\", \"\\\"\")\n",
        "raw_text = raw_text.replace(\"”\", \"\\\"\")\n",
        "raw_text = raw_text.replace(\"’\", \"'\")\n",
        "raw_text = raw_text.replace(\"‘\", \"'\")\n",
        "\n",
        "print(raw_text[:300])"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Alice's Adventures in Wonderland by Lewis Carroll CHAPTER I. Down the Rabbit-Hole  Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do: once or twice she had peeped into the book her sister was reading, but it had no pictures or conversations in it\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b9q9fMtYkZ9W"
      },
      "source": [
        "from konlpy.tag import Okt\n",
        "\n",
        "\n",
        "BERT_SUBTOKEN_PREFIX = \"##\"\n",
        "\n",
        "class KonlpyTokenizer():\n",
        "\n",
        "  def __init__(self):\n",
        "    self._tokens = []\n",
        "    self._tokenizer = Okt()\n",
        "\n",
        "  # texts = '하늘이 푸른가요? 나는 푸른색이 좋아요'\n",
        "  # return ['하늘', '##이', '푸른가요', '?', '나', '##는', '푸른색', '##이', '좋아요']\n",
        "  def tokenize(self, text):\n",
        "\n",
        "    def _has_preceding_space(text, token, last_position):\n",
        "      # print(text[last_position:last_position+20],\"-----\",token)\n",
        "      return text[last_position:last_position+20].startswith(\" \"+token)\n",
        "\n",
        "    poses = self._tokenizer.pos(text)\n",
        "\n",
        "    tokens = []\n",
        "    last_position = 0\n",
        "    for i, pos in enumerate(poses):\n",
        "      org_token = pos[0]\n",
        "      token = pos[0]\n",
        "      if i==0:\n",
        "        pass\n",
        "      elif pos[1]==\"Punctuation\":\n",
        "        if _has_preceding_space(text, token, last_position):  # \" 'of\"\n",
        "          last_position += 1\n",
        "        else:                                                 # \"습니다.\"\n",
        "          pass\n",
        "        if len(token)!=1:\n",
        "          token = list(token)\n",
        "      elif _has_preceding_space(text, token, last_position):\n",
        "        last_position += 1\n",
        "      elif pos[1] in [\"Alpha\", \"Foreign\", \"Number\", \"URL\"]:\n",
        "        pass\n",
        "      else:\n",
        "        token = BERT_SUBTOKEN_PREFIX+token \n",
        "      if type(token)==list:\n",
        "        tokens.extend(token)\n",
        "      else:\n",
        "        tokens.append(token)\n",
        "      last_position += len(org_token)\n",
        "\n",
        "    return tokens\n",
        "\n",
        "# t = \"CHAPTER I. Down the Rabbit-Hole  Alice was beginning to get \\\"very\\\" tired of sitting by her sister on the bank,\"\n",
        "# t = \", \\\"_ You are old , Father William _,'\\\" said the Caterpillar . Alice folded her hands , and began : — \\\" You are old , Father William ,\\\"\"\n",
        "# t = \"at www.gutenberg.org . If you are not located in the United States , you will have to check the laws of the country where you are located before using this\"\n",
        "\n",
        "# konlply_tokenizer = KonlpyTokenizer()\n",
        "# tokenized = konlply_tokenizer.tokenize(t)\n",
        "# print(tokenized)\n",
        "\n"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LorQArLIXGQp",
        "outputId": "a1767939-43fb-48d6-821d-5326d6466a10"
      },
      "source": [
        "konlply_tokenizer = KonlpyTokenizer()\n",
        "tokenized_text = konlply_tokenizer.tokenize(raw_text)\n",
        "print(raw_text[:103])\n",
        "print(tokenized_text[:20])"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Alice's Adventures in Wonderland by Lewis Carroll CHAPTER I. Down the Rabbit-Hole  Alice was beginning \n",
            "['Alice', \"'\", 's', 'Adventures', 'in', 'Wonderland', 'by', 'Lewis', 'Carroll', 'CHAPTER', 'I', '.', 'Down', 'the', 'Rabbit', '-', 'Hole', 'Alice', 'was', 'beginning']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aYddCOKBazWS"
      },
      "source": [
        "## 샘플링"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iBWjkR4GXGNi"
      },
      "source": [
        "SEQUENCE_LENGTH = 30\n",
        "OFFSET = 1"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Iqs6CyOXGKh"
      },
      "source": [
        "input_tokens = []\n",
        "output_tokens = []\n",
        "for i in range(len(tokenized_text)):\n",
        "  if i+SEQUENCE_LENGTH+OFFSET >= len(tokenized_text): break\n",
        "  input_tokens.append(tokenized_text[i:i+SEQUENCE_LENGTH])\n",
        "  output_tokens.append(tokenized_text[i+SEQUENCE_LENGTH+OFFSET-1])"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N8T3Y3SYa7JQ",
        "outputId": "0d703b4f-0949-48ad-d809-bcdf95d20382"
      },
      "source": [
        "print(len(input_tokens))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "39380\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SYHOlbEsa4rM"
      },
      "source": [
        "## 데이터 섞기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5qb7KKoMbHK3"
      },
      "source": [
        "input_tokens = np.array(input_tokens, dtype=str)\n",
        "output_tokens = np.array(output_tokens, dtype=str)\n",
        "\n",
        "indexes = np.arange(input_tokens.shape[0])\n",
        "np.random.shuffle(indexes)\n",
        "\n",
        "input_tokens = input_tokens[indexes].tolist()\n",
        "output_tokens = output_tokens[indexes].tolist()\n"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4V2EOtoMdj3x"
      },
      "source": [
        "## Vocab 파일 만들기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vzl1bSff8vyK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "87f16e01-9d52-4206-e6cf-b2a0ee2ef7f0"
      },
      "source": [
        "import collections\n",
        "from collections import OrderedDict\n",
        "\n",
        "\n",
        "class KonlpyVocabMaker():\n",
        "\n",
        "  def __init__(self, texts):\n",
        "    self._tokens = []\n",
        "    self._tokenize(texts)\n",
        "\n",
        "  # texts = '하늘이 푸른가요? 나는 푸른색이 좋아요'\n",
        "  # return ['하늘', '##이', '푸른가요', '?', '나', '##는', '푸른색', '##이', '좋아요']\n",
        "  def _tokenize(self, texts):\n",
        "\n",
        "    konlply_tokenizer = KonlpyTokenizer()\n",
        "\n",
        "    # 각 문장별로 토크나이징\n",
        "    all_tokens = []  \n",
        "    if type(texts)==str: texts = [texts]\n",
        "    for text in tqdm(texts):\n",
        "      all_tokens.extend(konlply_tokenizer.tokenize(text))\n",
        "\n",
        "    # 빈도 순으로 정열\n",
        "    counts = collections.Counter(all_tokens)\n",
        "    sorted_tokens = sorted(all_tokens, key=counts.get, reverse=True)\n",
        "\n",
        "    # 단어 중복 삭제\n",
        "    sorted_tokens = list(OrderedDict.fromkeys(sorted_tokens))\n",
        "\n",
        "    # Bert의 4개 특수 토큰을 삽입\n",
        "    sorted_tokens.insert(0, '[PAD]')\n",
        "    sorted_tokens.insert(1, '[UNK]')\n",
        "    sorted_tokens.insert(2, '[CLS]')\n",
        "    sorted_tokens.insert(3, '[SEP]')\n",
        "    sorted_tokens.insert(4, '[MSK]')\n",
        "\n",
        "    self._tokens = sorted_tokens\n",
        "\n",
        "  def get_vocab(self):\n",
        "    return self._tokens\n",
        "\n",
        "\n",
        "# t = \"CHAPTER I. Down the Rabbit-Hole  Alice was beginning to get \\\"very\\\" tired of sitting by her sister on the bank,\"\n",
        "# konlply_tokenizer = KonlpyTokenizer()\n",
        "# tokenized = konlply_tokenizer.tokenize(t)\n",
        "# print(tokenized)\n",
        "\n",
        "t = \", \\\"_ You are old , Father William _,'\\\" said the Caterpillar . Alice folded her hands , and began : — \\\" You are old , Father William ,\\\"\"\n",
        "konlply_vocab_maker = KonlpyVocabMaker(t)\n",
        "v = konlply_vocab_maker.get_vocab()\n",
        "print(v)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:00<00:00, 830.72it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MSK]', ',', '\"', '_', 'You', 'are', 'old', 'Father', 'William', \"'\", 'said', 'the', 'Caterpillar', '.', 'Alice', 'folded', 'her', 'hands', 'and', 'began', ':', '—']\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "68t7efMsiy_v"
      },
      "source": [
        "### vocab 만들기 실행"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NiPuK2kJiy_v",
        "outputId": "b4bccf9b-9012-49fd-e972-97ac3acc7899"
      },
      "source": [
        "konlply_vocab_maker = KonlpyVocabMaker(raw_text)\n",
        "\n",
        "vocab = konlply_vocab_maker.get_vocab()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:01<00:00,  1.24s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2FgGzgCzD7DE",
        "outputId": "1e1678d6-68bb-4575-f8c9-fcda98bc277b"
      },
      "source": [
        "print(\"org vocab size =\",len(vocab))\n",
        "vocab = vocab[:MAX_VOCAB_SIZE]\n",
        "vocab_size = len(vocab)\n",
        "print(\"vocab_size = \", len(vocab))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "org vocab size = 3412\n",
            "vocab_size =  3412\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BP4ABKZyD7Ah",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e28d731a-ef99-4bba-93ab-14b4b2997d23"
      },
      "source": [
        "print(vocab[:20])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MSK]', ',', '\"', 'the', '.', 'and', 'to', \"'\", 'a', 'of', 'I', 'it', 'she', 'said', '!', '_']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pq5F4g-odzD-"
      },
      "source": [
        "### vocab 파일 저장"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rRcTb42jdh-W"
      },
      "source": [
        "with open(CUSTOM_VOCAB_FILE, 'w') as f:\n",
        "  for item in vocab:\n",
        "    f.write(\"%s\\n\" % item)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t9rV-Dump0bW",
        "outputId": "cace2ded-2b3f-4395-d971-8a486a9813e8"
      },
      "source": [
        "!wc {CUSTOM_VOCAB_FILE}"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " 3412  3412 24370 custom_vocab.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ACEb0EydeJK5"
      },
      "source": [
        "## Bert Tokenizer 생성"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "astIVj9-dh-X"
      },
      "source": [
        "bert_tokenizer = BertTokenizer(vocab_file=CUSTOM_VOCAB_FILE, do_lower_case=False, model_max_length=SEQ_LENGTH)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jHKzG8c5dh-Y",
        "outputId": "2308eaaf-48e3-4378-f2ef-f1443e0fdac9"
      },
      "source": [
        "bert_tokenized = bert_tokenizer(input_tokens[100], max_length=BERT_SEQUENCE_LENGTH, padding='max_length', is_pretokenized=True, )\n",
        "print(\"vocab              :\", vocab[:20])\n",
        "print(\"original sentence  :\", input_tokens[100])\n",
        "print(\"tokens             :\", bert_tokenizer.convert_ids_to_tokens(bert_tokenized['input_ids']))\n",
        "print(\"token id           :\", bert_tokenized['input_ids'])\n",
        "print(\"attention mask     :\", bert_tokenized['attention_mask'])\n",
        "print(\"token type         :\", bert_tokenized['token_type_ids'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "vocab              : ['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MSK]', ',', '\"', 'the', '.', 'and', 'to', \"'\", 'a', 'of', 'I', 'it', 'she', 'said', '!', '_']\n",
            "original sentence  : ['I', 'don', \"'\", 't', 'like', 'them', 'raw', '.', '\"', '\"', 'Well', ',', 'be', 'off', ',', 'then', '!', '\"', 'said', 'the', 'Pigeon', 'in', 'a', 'sulky', 'tone', ',', 'as', 'it', 'settled', 'down']\n",
            "tokens             : ['[CLS]', 'I', 'don', \"'\", 't', 'like', 'them', 'raw', '.', '\"', '\"', 'Well', ',', 'be', 'off', ',', 'then', '!', '\"', 'said', 'the', 'Pigeon', 'in', 'a', 'sulky', 'tone', ',', 'as', 'it', 'settled', 'down', '[SEP]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
            "token id           : [2, 14, 114, 11, 30, 62, 59, 2353, 8, 6, 6, 221, 5, 39, 94, 5, 79, 18, 6, 17, 7, 414, 20, 12, 1090, 149, 5, 26, 15, 1176, 57, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
            "attention mask     : [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
            "token type         : [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kfn4wVkQhkx4"
      },
      "source": [
        "![bert_input_architecture](https://user-images.githubusercontent.com/1250095/50039788-8e4e8a00-007b-11e9-9747-8e29fbbea0b3.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cEnAfwf_Vb1I"
      },
      "source": [
        "## x, y 생성"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k4SlXQh2LwfI"
      },
      "source": [
        "\n",
        "tokenizer 사용 중에 경고 메시지가 많이 뜬다. 억제한다.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "twrpGcuELwfS"
      },
      "source": [
        "import logging\n",
        "logging.basicConfig(level=logging.ERROR)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S7I3pUuBq7Ak"
      },
      "source": [
        "from random import randrange\n",
        "\n",
        "def build_model_input_output(input_tokens, output_tokens):\n",
        "  input_ids = []\n",
        "  attention_masks = []\n",
        "  token_type_ids = []\n",
        "  labels = []\n",
        "\n",
        "  for input_token, output_token in zip(input_tokens, output_tokens):\n",
        "    bert_tokenized = bert_tokenizer(input_token, max_length=BERT_SEQUENCE_LENGTH, padding='max_length', is_pretokenized=True)\n",
        "    # bert_tokenized = {'input_ids': [101, ...], 'token_type_ids': [0, ...], 'attention_mask': [1, ...]}\n",
        "    input_ids.append(bert_tokenized['input_ids'][:BERT_SEQUENCE_LENGTH]) # 버그인지 몰라도 max_length 이상이어도 더 크게 나온다.\n",
        "    attention_masks.append(bert_tokenized['attention_mask'][:BERT_SEQUENCE_LENGTH])\n",
        "    token_type_ids.append(bert_tokenized['token_type_ids'][:BERT_SEQUENCE_LENGTH])\n",
        "    labels.append(vocab.index(output_token))\n",
        "\n",
        "  return (np.array(input_ids), np.array(attention_masks), np.array(token_type_ids)), np.array(labels)\n",
        "\n",
        "# MAX_DATA_COUNT = 9\n",
        "# x, y = build_model_input_output(input_tokens[:MAX_DATA_COUNT], output_tokens[:MAX_DATA_COUNT])  \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "liooOUBn1Cgu"
      },
      "source": [
        "MAX_DATA_COUNT = 10000 * 10\n",
        "x, y = build_model_input_output(input_tokens[:MAX_DATA_COUNT], output_tokens[:MAX_DATA_COUNT])  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2HfIzDjeY0d7"
      },
      "source": [
        "## train/test 분리"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xxqohOmDa72g"
      },
      "source": [
        "def split_bert_data(x, y, test_ratio):\n",
        "  split_index = int(len(y)*(1-test_ratio))\n",
        "  train_x = (x[0][:split_index], x[1][:split_index], x[2][:split_index])\n",
        "  test_x  = (x[0][split_index:], x[1][split_index:], x[2][split_index:])\n",
        "  train_y, test_y = y[:split_index], y[split_index:]\n",
        "\n",
        "  return (train_x, train_y), (test_x, test_y)\n",
        "\n",
        "(train_x, train_y), (test_x, test_y) = split_bert_data(x, y, test_ratio=0.2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-cUiOQOUhYcW",
        "outputId": "c965397b-7346-4802-c087-a9b35e49e60f"
      },
      "source": [
        "print(bert_tokenizer.decode(train_x[0][0][:35]))\n",
        "print(train_x[0][0][:35])\n",
        "print(train_x[1][0][:35])\n",
        "print(train_x[2][0][:35])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CLS] and she told her sister, as well as she could remember them, all these strange Adventures of hers that you have just been reading about ; and when [SEP] [PAD] [PAD] [PAD]\n",
            "[   2    9   16  686   27  459    5   26  144   26   16   73  319   59\n",
            "    5   36  308  752 1006   13  969   24   21   67  131  154  833   54\n",
            "   37    9   83    3    0    0    0]\n",
            "[1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0]\n",
            "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HWb-Ip8pYEQg"
      },
      "source": [
        "# 학습"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Txqhf6XYFyP"
      },
      "source": [
        "## 모델 생성"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7-vXh3QmT5nu"
      },
      "source": [
        "from tensorflow.keras.initializers import TruncatedNormal\n",
        "from tensorflow.keras.layers import Dense, Dropout\n",
        "\n",
        "class TFBertClassifier(tf.keras.Model):\n",
        "  def __init__(self):\n",
        "    super(TFBertClassifier, self).__init__()\n",
        "\n",
        "    self.bert = TFBertModel.from_pretrained(BERT_MODEL_NAME, trainable=True)\n",
        "    self.dropout = Dropout(self.bert.config.hidden_dropout_prob)\n",
        "    self.classifier = Dense(vocab_size, kernel_initializer=TruncatedNormal(self.bert.config.initializer_range), \n",
        "                            name=\"classifier\", activation=\"softmax\")\n",
        "\n",
        "  def call(self, inputs, training=True):\n",
        "\n",
        "    # outputs = self.bert(inputs, attention_mask=attention_mask, token_type_ids=token_type_ids)\n",
        "    outputs = self.bert(inputs)\n",
        "    # outputs 값: # sequence_output, pooled_output, (hidden_states), (attentions)\n",
        "    pooled_output = outputs[1]\n",
        "    # pooled_output = outputs[1] \n",
        "    v = self.dropout(pooled_output, training=training)\n",
        "    out = self.classifier(v)\n",
        "\n",
        "    return out\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GfycyrGDYg51"
      },
      "source": [
        "참고로 Bert의 default 설정은 다음과 같다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6rFMC5hjYe93",
        "outputId": "7278a0ec-db42-43cc-dc75-7c599f9827a8"
      },
      "source": [
        "print(model.bert.config)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BertConfig {\n",
            "  \"architectures\": [\n",
            "    \"BertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"directionality\": \"bidi\",\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"pooler_fc_size\": 768,\n",
            "  \"pooler_num_attention_heads\": 12,\n",
            "  \"pooler_num_fc_layers\": 3,\n",
            "  \"pooler_size_per_head\": 128,\n",
            "  \"pooler_type\": \"first_token_transform\",\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"vocab_size\": 119547\n",
            "}\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ZIUqxuBbfq4"
      },
      "source": [
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.losses import SparseCategoricalCrossentropy\n",
        "\n",
        "optimizer = Adam(3e-5)\n",
        "loss = SparseCategoricalCrossentropy()\n",
        "model.compile(optimizer=optimizer, loss=loss, metrics=[\"accuracy\"])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s5JdxQn-bNPp"
      },
      "source": [
        "## 학습 실행"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f2yQcJaXkNcT",
        "outputId": "919f8586-9609-4154-b261-b6092877d0a9"
      },
      "source": [
        "print(train_y.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(31504,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6WU1NbHObNBY",
        "outputId": "bf8022f7-fa19-4afb-8d5f-e0dfa97be4f2"
      },
      "source": [
        "history = model.fit(train_x, train_y, epochs=1, batch_size=32, validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "887/887 [==============================] - 467s 510ms/step - loss: 6.0527 - accuracy: 0.0941 - val_loss: 5.6322 - val_accuracy: 0.1346\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9JlJLgKw0iHT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e68612b3-ac4c-4d4e-a8c2-153b80f24602"
      },
      "source": [
        "loss, acc = model.evaluate(test_x, test_y, batch_size=32)\n",
        "print(\"loss =\", loss)\n",
        "print(\"acc =\", acc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "247/247 [==============================] - 42s 169ms/step - loss: 5.6382 - accuracy: 0.1342\n",
            "loss = 5.638205528259277\n",
            "acc = 0.1342051774263382\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PvnNncG_3p87"
      },
      "source": [
        "## 분류 실행"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RNsQMfcjV5T8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c8ee00e1-9552-4c45-a93b-4f231533eab9"
      },
      "source": [
        "def do_predict(test_input, test_output):\n",
        "  y_ = model.predict(test_input)\n",
        "  predicted = y_[0].argsort()[-5:][::-1]\n",
        "  org_input = bert_tokenizer.decode(test_input[0][0][:35])\n",
        "  org_input = org_input.replace(\"[CLS]\", \"\")\n",
        "  org_input = org_input.replace(\"[SEP]\", \"\")\n",
        "  org_input = org_input.replace(\"[PAD]\", \"\")\n",
        "  org_input = org_input.replace(\"[PAD]\", \"\")\n",
        "  org_input = org_input.replace(\" \", \" \")\n",
        "  print( org_input, \", --> TRUTH :\", vocab[test_output],  \", EXPECTED :\", [vocab[i] for i in predicted])\n",
        "\n",
        "for i in range(5):\n",
        "  do_predict((test_x[0][i:i+1], test_x[1][i:i+1], test_x[2][i:i+1]), test_y[i])\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " into a conversation. \" You don't know much, \" said the Duchess ; \" and that's a fact. \" Alice did not at     , --> TRUTH : all , EXPECTED : [',', 'the', \"'\", '.', 'a']\n",
            " distributing, performing, displaying or creating derivative works based on the work as long as all references to Project Gutenberg are removed. Of course, we hope that     , --> TRUTH : you , EXPECTED : [',', 'the', '.', 'a', \"'\"]\n",
            " the King had said that day. \" That _ proves _ his guilt, \" said the Queen. \" It proves nothing of the sort! \" said     , --> TRUTH : Alice , EXPECTED : ['the', ',', \"'\", 'a', 'you']\n",
            " without speaking, but at last it unfolded its arms, took the hookah out of its mouth again, and said, \" So you think you're     , --> TRUTH : changed , EXPECTED : ['the', 'a', \"'\", 'she', 'be']\n",
            " s making personal remarks now? \" the Hatter asked triumphantly. Alice did not quite know what to say to this : so she helped herself to some tea     , --> TRUTH : and , EXPECTED : [',', 'of', '.', 'to', '!']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DibdK35ktIy9"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}