preprocess_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "colab": {
      "name": "preprocess_data.ipynb",
      "provenance": []
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e90d7ed8b7fb4546b199d7681b4cfd99": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6eacce7e9cf84e61a402764375a81bfd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7f18db280e5f405ab3ef43bd86c7a656",
              "IPY_MODEL_233f6f1c09164a4b879627e49b770a50"
            ]
          }
        },
        "6eacce7e9cf84e61a402764375a81bfd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7f18db280e5f405ab3ef43bd86c7a656": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_715887b05ace421693bd31844d9411b0",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 405234788,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 405234788,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d49d8d74c88740e6a52dd40ebd610632"
          }
        },
        "233f6f1c09164a4b879627e49b770a50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1edb2e71a4dc48b884c54149c7ea5799",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 405M/405M [00:50&lt;00:00, 7.97MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7c98c19444c04f3d83209e758ad78d9f"
          }
        },
        "715887b05ace421693bd31844d9411b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d49d8d74c88740e6a52dd40ebd610632": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1edb2e71a4dc48b884c54149c7ea5799": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7c98c19444c04f3d83209e758ad78d9f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v5D6TsQlvSZH"
      },
      "source": [
        "# [Data Preprocessing](http://colab.research.google.com/github/boringPpl/presidential_debates_comments_clustering/blob/main/preprocess_data.ipynb)\n",
        "\n",
        "## 1. Setup\n",
        "\n",
        "### 1.1 Hardware\n",
        "\n",
        "First, check that the runtime in Google Colab is set to GPU. If it is not, go to **Runtime > Change runtime type** and change the **Hardware Accelerator** to **GPU**."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IsSzCZyxy9As",
        "outputId": "ed0d65df-e57c-4695-97f5-2f8f53eb6f7f"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mon Mar 22 12:13:17 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.56       Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   33C    P8    27W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7f99sNEh2bnr"
      },
      "source": [
        "The following shows the CPU details for the Google Colab instance"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "klKuW-xB2FKl",
        "outputId": "39d12fac-688a-482a-e20c-3fa3881cc9f3"
      },
      "source": [
        "!lscpu"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Architecture:        x86_64\n",
            "CPU op-mode(s):      32-bit, 64-bit\n",
            "Byte Order:          Little Endian\n",
            "CPU(s):              2\n",
            "On-line CPU(s) list: 0,1\n",
            "Thread(s) per core:  2\n",
            "Core(s) per socket:  1\n",
            "Socket(s):           1\n",
            "NUMA node(s):        1\n",
            "Vendor ID:           GenuineIntel\n",
            "CPU family:          6\n",
            "Model:               63\n",
            "Model name:          Intel(R) Xeon(R) CPU @ 2.30GHz\n",
            "Stepping:            0\n",
            "CPU MHz:             2299.998\n",
            "BogoMIPS:            4599.99\n",
            "Hypervisor vendor:   KVM\n",
            "Virtualization type: full\n",
            "L1d cache:           32K\n",
            "L1i cache:           32K\n",
            "L2 cache:            256K\n",
            "L3 cache:            46080K\n",
            "NUMA node0 CPU(s):   0,1\n",
            "Flags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat md_clear arch_capabilities\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iN3t5qwA2rbb"
      },
      "source": [
        "The memory available for the instance is listed as follows"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "46fl6M1z2IVk",
        "outputId": "ae1548eb-6306-49a4-8ecf-f8d10c79b49e"
      },
      "source": [
        "!free -h"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              total        used        free      shared  buff/cache   available\n",
            "Mem:            12G        562M         10G        1.0M        2.0G         11G\n",
            "Swap:            0B          0B          0B\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6FT94kk61QKd"
      },
      "source": [
        "### 1.2 Install Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "49kLid8avSZJ",
        "outputId": "e0e00fa3-d6ee-4c5d-9621-11d77d3dd6c8"
      },
      "source": [
        "!git clone https://github.com/boringPpl/presidential_debates_comments_clustering.git\n",
        "%cd presidential_debates_comments_clustering\n",
        "!pip install -qr requirements.txt"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'presidential_debates_comments_clustering'...\n",
            "remote: Enumerating objects: 111, done.\u001b[K\n",
            "remote: Counting objects: 100% (111/111), done.\u001b[K\n",
            "remote: Compressing objects: 100% (86/86), done.\u001b[K\n",
            "remote: Total 111 (delta 50), reused 64 (delta 22), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (111/111), 11.86 MiB | 18.71 MiB/s, done.\n",
            "Resolving deltas: 100% (50/50), done.\n",
            "/content/presidential_debates_comments_clustering\n",
            "\u001b[K     |████████████████████████████████| 133kB 4.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 6.4MB 5.7MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[K     |████████████████████████████████| 81kB 7.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 36.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.2MB 40.7MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 36.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.0MB 39.5MB/s \n",
            "\u001b[?25h  Building wheel for hdbscan (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ZevAggEvSZK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "e90d7ed8b7fb4546b199d7681b4cfd99",
            "6eacce7e9cf84e61a402764375a81bfd",
            "7f18db280e5f405ab3ef43bd86c7a656",
            "233f6f1c09164a4b879627e49b770a50",
            "715887b05ace421693bd31844d9411b0",
            "d49d8d74c88740e6a52dd40ebd610632",
            "1edb2e71a4dc48b884c54149c7ea5799",
            "7c98c19444c04f3d83209e758ad78d9f"
          ]
        },
        "outputId": "7bf7b3ad-f173-4cc5-cf2f-629c5af39a11"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import emoji\n",
        "\n",
        "from sentence_transformers import SentenceTransformer\n",
        "from sklearn.cluster import KMeans\n",
        "\n",
        "embedder = SentenceTransformer('bert-base-nli-mean-tokens')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e90d7ed8b7fb4546b199d7681b4cfd99",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=405234788.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eGSugYCGvSZK"
      },
      "source": [
        "## 2. Exploratory Data Analysis\n",
        "\n",
        "### 2.1 Load data into Pandas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BWRl3_GzvSZK"
      },
      "source": [
        "def load_comments(video_id):\n",
        "    filename = f'data/{video_id}_csv_final.csv'\n",
        "    df = pd.read_csv(filename, index_col=0)\n",
        "    return df"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OLffUHsTvSZL"
      },
      "source": [
        "df1 = load_comments('bPiofmZGb8o')\n",
        "comments1 = df1['Comments']\n",
        "df1['Updated At'] = pd.to_datetime(df1['Updated At'], format='%Y-%m-%dT%H:%M:%SZ', errors='coerce')"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eNpUIfUnvSZL"
      },
      "source": [
        "### 2.2 Exploratory data analysis"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "bnHLsvy7vSZL",
        "outputId": "69e2f993-05a6-41cd-e5ae-c93c7d1e37e4"
      },
      "source": [
        "print(f'The shape of the dataframe is: {df1.shape}')\n",
        "print(f\"The time range for the data is: {df1['Updated At'].min():%Y-%m-%d %H%Mh} to {df1['Updated At'].max():%Y-%m-%d %H%Mh}\")\n",
        "df1.head()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The shape of the dataframe is: (49608, 6)\n",
            "The time range for the data is: 2020-10-23 0243h to 2021-01-18 0945h\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Comments</th>\n",
              "      <th>Comment ID</th>\n",
              "      <th>Reply Count</th>\n",
              "      <th>Like Count</th>\n",
              "      <th>Updated At</th>\n",
              "      <th>Viewer Rating</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>25:28\\nTrump: Did you hear anything?\\nBiden: N...</td>\n",
              "      <td>UgxJhAbG8Z-yTkfAPPl4AaABAg</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2021-01-18 09:45:30</td>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>53:39 trump the idiot.\\nYour the big man! I do...</td>\n",
              "      <td>UgyHNoBZEukz5n3FHgR4AaABAg</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2021-01-18 03:57:03</td>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Trump is the President of the USA !  no biden...</td>\n",
              "      <td>UgzhlTI8JkT8n_sKDuR4AaABAg</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2021-01-18 03:36:51</td>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Driving home from school today because that We...</td>\n",
              "      <td>UgxJmfHfPpwcEdkLFNV4AaABAg</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2021-01-18 02:03:25</td>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Trump won election the swamp rats in Washingto...</td>\n",
              "      <td>UgwZT9ggKbgHu6KbOgN4AaABAg</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2021-01-18 00:44:14</td>\n",
              "      <td>none</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            Comments  ... Viewer Rating\n",
              "0  25:28\\nTrump: Did you hear anything?\\nBiden: N...  ...          none\n",
              "1  53:39 trump the idiot.\\nYour the big man! I do...  ...          none\n",
              "2   Trump is the President of the USA !  no biden...  ...          none\n",
              "3  Driving home from school today because that We...  ...          none\n",
              "4  Trump won election the swamp rats in Washingto...  ...          none\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 320
        },
        "id": "2lhHs12yvSZL",
        "outputId": "a7448c4f-618a-46dd-e431-67dc3693f011"
      },
      "source": [
        "ax = df1['Updated At'].hist(bins=25, figsize=(12, 5))\n",
        "ax.set_yscale('log')"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsUAAAEvCAYAAACt/LxhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASmklEQVR4nO3df4xl51kf8O+DjRHy0inU0Tay3Y7RWqhulrZkmgiE0GxRlXXMEkCWateKsJRkFSSrRfiPLqKC/rlUAgmktNHKsQIS8jYNiHq9DmmLWAUJSG2nIbaxEky0KN6mMSEwsFGkdMXTP/YmDMvM7uzO3D137vv5SCPf+8758cw896y/e/Y951R3BwAARvYNUxcAAABTE4oBABieUAwAwPCEYgAAhicUAwAwPKEYAIDh3Tp1AUlyxx139Orq6tRlDOPLX/5ybr/99qnLGJ4+7G/6Nz09WF56O71l7sELL7zwxe5+w5XjCxGKV1dX8/zzz09dxjDOnTuX9fX1qcsYnj7sb/o3PT1YXno7vWXuQVX98Vbjpk8AADA8oRgAgOEJxQAADG/SUFxVx6rq1MbGxpRlAAAwuElDcXef6e7jKysrU5YBAMDgTJ8AAGB4QjEAAMMTigEAGJ5QDADA8IRiAACGN+ljnqvqWJJjhw4dmmT/qyfOzn0f508+MPd9AACwO27JBgDA8EyfAABgeEIxAADDE4oBABieUAwAwPCEYgAAhicUAwAwPKEYAIDhCcUAAAxv0lBcVceq6tTGxsaUZQAAMDhPtAMAYHimTwAAMDyhGACA4QnFAAAMTygGAGB4QjEAAMMTigEAGJ5QDADA8IRiAACGJxQDADA8oRgAgOEJxQAADE8oBgBgeJOG4qo6VlWnNjY2piwDAIDBTRqKu/tMdx9fWVmZsgwAAAZn+gQAAMMTigEAGJ5QDADA8IRiAACGJxQDADA8oRgAgOEJxQAADE8oBgBgeEIxAADDE4oBABieUAwAwPCEYgAAhicUAwAwPKEYAIDhCcUAAAxPKAYAYHhCMQAAwxOKAQAYnlAMAMDw5hKKq+r2qnq+qn5gHtsHAIC9tKNQXFVPVtXrVfXSFeNHq+rTVfVqVZ3Y9K1/l+RDe1koAADMy07PFH8wydHNA1V1S5L3Jbk/yX1JHq6q+6rqXyb5gySv72GdAAAwN9XdO1uwajXJM939ptn7707yH7r7bbP3Pzlb9ECS23M5KH8lyQ93919tsb3jSY4nycGDB998+vTpXf0gN+LFCxtz38fhO1fmvo/rdfHixRw4cGDqMoanD/ub/k1PD5aX3k5vmXtw5MiRF7p77crxW3exzTuTfG7T+9eSvLW7H0uSqno0yRe3CsRJ0t2nkpxKkrW1tV5fX99FKTfm0RNn576P84+sz30f1+vcuXOZ4vfN36QP+5v+TU8PlpfeTm/EHuwmFF9Vd39wXtsGAIC9tJu7T1xIcvem93fNxgAAYF/ZTSh+Lsm9VXVPVd2W5KEkT1/PBqrqWFWd2tiY/9xeAADYzk5vyfZUkt9N8h1V9VpVvau7LyV5LMlHk7yS5EPd/fL17Ly7z3T38ZWVxbsYDQCAcexoTnF3P7zN+LNJnt3TigAA4CbzmGcAAIY3aSg2pxgAgEUwaSg2pxgAgEVg+gQAAMMTigEAGJ5QDADA8FxoBwDA8FxoBwDA8EyfAABgeEIxAADDE4oBABieC+0AABieC+0AABie6RMAAAxPKAYAYHhCMQAAwxOKAQAYnrtPAAAwPHefAABgeKZPAAAwPKEYAIDhCcUAAAxPKAYAYHhCMQAAwxOKAQAYnvsUAwAwPPcpBgBgeKZPAAAwPKEYAIDhCcUAAAxPKAYAYHhCMQAAwxOKAQAYnlAMAMDwhGIAAIbniXYAAAzPE+0AABjerVMXsOxWT5yd6/bPn3xgrtsHABiBOcUAAAxPKAYAYHhCMQAAwxOKAQAYngvtuCYXCwIAy86ZYgAAhicUAwAwPKEYAIDhCcUAAAxPKAYAYHhCMQAAw5s0FFfVsao6tbGxMWUZAAAMbtJQ3N1nuvv4ysrKlGUAADA40ycAABieUAwAwPCEYgAAhicUAwAwPKEYAIDhCcUAAAxPKAYAYHhCMQAAwxOKAQAY3q1TF8DurJ44e93rPH74Uh69gfUAAJaVM8UAAAxPKAYAYHhCMQAAwxOKAQAYnlAMAMDwhGIAAIYnFAMAMDyhGACA4e15KK6qf1RV76+qD1fVj+319gEAYK/t6Il2VfVkkh9I8np3v2nT+NEkv5DkliRPdPfJ7n4lyXur6huS/HKS/7z3ZbNMbuSpfNfj/MkH5rp9AGD/2+mZ4g8mObp5oKpuSfK+JPcnuS/Jw1V13+x7P5jkbJJn96xSAACYkx2F4u7+WJIvXTH8liSvdvdnu/urSU4necds+ae7+/4kj+xlsQAAMA/V3TtbsGo1yTNfmz5RVQ8mOdrd7569f2eStyb5cJIfSfJNST7V3e/bZnvHkxxPkoMHD7759OnTu/pBbsSLFzZu+j4XwcFvTr7wlamruHkO37kydQlbunjxYg4cODB1Gdwg/ZueHiwvvZ3eMvfgyJEjL3T32pXjO5pTfD26+1yScztY7lSSU0mytrbW6+vre13KNT0657msi+rxw5fycy/ueesX1vlH1qcuYUvnzp3LFJ979ob+TU8PlpfeTm/EHuzm7hMXkty96f1dszEAANhXdhOKn0tyb1XdU1W3JXkoydN7UxYAANw8OwrFVfVUkt9N8h1V9VpVvau7LyV5LMlHk7yS5EPd/fL17LyqjlXVqY2NMef2AgCwGHY0sbS7H95m/Nns4rZr3X0myZm1tbX33Og2AABgtzzmGQCA4QnFAAAMb9JQbE4xAACLYNJQ3N1nuvv4yspiPlwBAIAxmD4BAMDwhGIAAIYnFAMAMDwX2gEAMDwX2gEAMDzTJwAAGJ5QDADA8IRiAACG50I7AACG50I7AACGZ/oEAADDE4oBABieUAwAwPCEYgAAhufuEwAADM/dJwAAGJ7pEwAADE8oBgBgeEIxAADDE4oBABieUAwAwPCEYgAAhuc+xQAADM99igEAGJ7pEwAADE8oBgBgeLdOXQDM2+qJs3Pfx/mTD8x9HwDA/DhTDADA8IRiAACGJxQDADA8oRgAgOEJxQAADM8T7QAAGJ4n2gEAMDzTJwAAGJ5QDADA8IRiAACGJxQDADA8oRgAgOEJxQAADE8oBgBgeEIxAADDE4oBABieUAwAwPCEYgAAhicUAwAwvElDcVUdq6pTGxsbU5YBAMDgJg3F3X2mu4+vrKxMWQYAAIO7deoCYBmsnjh73es8fvhSHt3heudPPnDd2wcAds6cYgAAhicUAwAwPKEYAIDhCcUAAAxPKAYAYHhCMQAAwxOKAQAYnlAMAMDwhGIAAIYnFAMAMDyhGACA4QnFAAAMTygGAGB4QjEAAMO7deoCgGtbPXF2rts/f/KBuW4fABadM8UAAAxPKAYAYHhzmT5RVT+U5IEkfyfJB7r7v89jPwAAsBd2fKa4qp6sqter6qUrxo9W1aer6tWqOpEk3f3r3f2eJO9N8q/2tmQAANhb1zN94oNJjm4eqKpbkrwvyf1J7kvycFXdt2mRfz/7PgAALKwdh+Lu/liSL10x/JYkr3b3Z7v7q0lOJ3lHXfazST7S3Z/Yu3IBAGDvVXfvfOGq1STPdPebZu8fTHK0u989e//OJG9N8pkkP5rkuSSf7O73b7Gt40mOJ8nBgwfffPr06V39IDfixQsbN32fi+DgNydf+MrUVbBIfTh858rUJew7Fy9ezIEDB6YuY2h6sLz0dnrL3IMjR4680N1rV47P5UK77v7FJL94jWVOJTmVJGtra72+vj6PUq7q0Tnf+3VRPX74Un7uRbeontoi9eH8I+tTl7DvnDt3LlP8ucVf04PlpbfTG7EHu/0/8oUkd296f9dsDOBv8AASABbZbu9T/FySe6vqnqq6LclDSZ7efVkAAHDz7PhMcVU9lWQ9yR1V9VqSn+nuD1TVY0k+muSWJE9298vXsc1jSY4dOnTo+qoG9tS8z+ICwKLbcSju7oe3GX82ybM3svPuPpPkzNra2ntuZH0AANgLHvMMAMDwhGIAAIY3aSiuqmNVdWpjY8z7BQMAsBgmDcXdfaa7j6+seHAAAADTMX0CAIDhCcUAAAzPnGIAAIZnTjEAAMMzfQIAgOHt+Il2AIts3o+qPn/ygbluH4BpOVMMAMDwhGIAAIbn7hMAAAzP3ScAABieC+0A2BPzvtgxccEjMD9CMQCwY+70wrISigEWhLABMB13nwAAYHiTnimuqmNJjh06dGjKMgCAgfhXGbYyaSju7jNJzqytrb1nyjoAAPbKMlx0+uKFjTw6x59jEf/iYPoEAADDc6EdwA5ceebn8cOX5noWha1t7sM8erCIZ6+Am8OZYgAAhudMMQDcJMsw1xSWlTPFAAAMTygGAGB4k4biqjpWVac2NjamLAMAgMG5TzHAIG7GfFamp89wY1xoBwAzAiWMy5xiAACGJxQDADA8oRgAgOGZUwwALIzVE2c9Rp1JCMUAAPvMvC8KffzwXDe/kEyfAABgeEIxAADD80Q7AACGN2ko7u4z3X18ZWVlyjIAABic6RMAAAxPKAYAYHhCMQAAwxOKAQAYnlAMAMDwhGIAAIYnFAMAMDyhGACA4VV3T11DqupPkvzx1HUM5I4kX5y6CPRhn9O/6enB8tLb6S1zD/5hd7/hysGFCMXcXFX1fHevTV3H6PRhf9O/6enB8tLb6Y3YA9MnAAAYnlAMAMDwhOIxnZq6AJLow36nf9PTg+Wlt9MbrgfmFAMAMDxnigEAGJ5QvACq6u6q+q2q+oOqermq/u1s/Nuq6n9U1R/O/vuts/FHqupTVfViVf1OVf2TTds6WlWfrqpXq+rEVfb5G1X151X1zBXjj83W7aq64yrr31NVH58t+1+q6rbZ+PdV1Seq6lJVPbjb383NtE/7sOVyVbVeVRtV9cnZ10/v5nez6Basd78yW/+lqnqyqr5xm/UdQ4vRB8fQDuxxb5+sqter6qVr7HPLz8B1/Pm4VMfYPu3B/jq+utvXxF9J3pjku2avvyXJZ5Lcl+Q/JjkxGz+R5Gdnr78nybfOXt+f5OOz17ck+aMk357ktiS/n+S+bfb5/UmOJXnmivF/lmQ1yfkkd1yl5g8leWj2+v1Jfmz2ejXJdyb55SQPTv27HaAPWy6XZP3KbS7z14L17u1Javb11NeOjS3WdwwtRh8cQzext7P335fku5K8dJX9bfsZ2K5nW2xjqY6xfdqDfXV8OVO8ALr78939idnrv0zySpI7k7wjyS/NFvulJD80W+Z3uvvPZuO/l+Su2eu3JHm1uz/b3V9Ncnq2ja32+ZtJ/nKL8f/d3eevVm9VVZJ/keTDW9R2vrs/leSvrvFjL5z91ofrWW7ZLVjvnu2ZJP9r07a/zjG0GH2YLecY2oE97G26+2NJvnSNXW77GRj1/1P7rQfXs9yiEIoXTFWt5vLfrD6e5GB3f372rf+b5OAWq7wryUdmr+9M8rlN33ttNrbX/l6SP+/uS3Pez2T2SR+u5bur6ver6iNV9Y8n2P8kFqV3s3+uf2eS39ji246hv22KPlzLkMfQteyytzu128/AUh9j+6QH17Jwx9etUxfAX6uqA0l+NcmPd/dfXP6L7mXd3VXVVyx/JJc/6N97UwtdckvSh0/k8mMsL1bV25P8epJ7J65p7hasd/8pyce6+7fnsO2FtiR9GPIYupYF6+2QlqQHC3l8OVO8IGZnM341ya9096/Nhr9QVW+cff+NSV7ftPx3JnkiyTu6+09nwxeS3L1ps3cluVBVb900mf0Hb7C+j87WfyLJnyb5u1X1tb9U3TXb9763z/qwre7+i+6+OHv9bJJvvNrFEMtgkXpXVT+T5A1JfmLTmGMoC9eHbY14DF3LHvV2u23fvam37802n4FrbGPpj7F91oNtLerx5UzxApjNffpAkle6++c3fevpJD+a5OTsv/9ttvw/SPJrSd7Z3Z/ZtPxzSe6tqnty+YP7UJJ/3d0vJ/mnu6mxu992Rc2/leTBXJ5j9PXa9rP92Ier/Cx/P8kXZmcN3pLLfwG+6h+I+9ki9a6q3p3kbUm+v7u/PmfRMbSYfbjK+kMdQ9eyh73dUnd/Lpt6Owuzf+szcI1tLPUxth97cJWfZTGPr16Aq/1G/8rlf9LoJJ9K8snZ19tzeU7Ubyb5wyT/M8m3zZZ/IsmfbVr2+U3bensuX5H6R0l+6ir7/O0kf5LkK7k8T+hts/F/M3t/Kcn/SfLENut/ey5fvPJqkv+a5Jtm4/98tv6Xc/kD/vLUv98l78OWyyV5LMnLuXy18O8l+Z6pf78D9e7SbN2vbfunt1nfMbQYfXAM3fzePpXk80n+3+x3/65t9rnlZ2C7nm2x/lIdY/u0B/vq+PJEOwAAhmdOMQAAwxOKAQAYnlAMAMDwhGIAAIYnFAMAMDyhGACA4QnFAAAMTygGAGB4/x9tQIkJZvi09gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 864x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QZyIA-tfvSZM",
        "outputId": "6dceebbd-9199-4dab-d73d-cf93ee8b632c"
      },
      "source": [
        "import emoji\n",
        "print(emoji.demojize('trending 😉\t'))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "trending :winking_face:\t\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lej-nAeXvSZM"
      },
      "source": [
        "df_corpus = df1"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UWq6agYPvSZM"
      },
      "source": [
        "df_corpus.rename(columns={'Comments': 'comment_text'}, inplace=True)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nvynIneBvSZN"
      },
      "source": [
        "# De-emojize\n",
        "df_corpus['comments_cleaned'] = df_corpus['comment_text'].apply(emoji.demojize)\n",
        "\n",
        "# Replace the colons, and \\n with a space\n",
        "df_corpus['comments_cleaned'] = df_corpus['comments_cleaned'].str.replace('[\\n:]', ' ', regex=True)\n",
        "df_corpus['comments_cleaned'] = df_corpus['comments_cleaned'].str.replace(r'\\\\n', ' ', regex=True)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9iwHuvFPvSZN"
      },
      "source": [
        "df_corpus['comments_cleaned'] = df_corpus['comments_cleaned'].str.lower()"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9YxEPBoTvSZN",
        "outputId": "8923d87d-c438-4e21-f41c-3c85a0a1071a"
      },
      "source": [
        "df_corpus.drop_duplicates(subset=['comments_cleaned'], inplace = True)\n",
        "df_corpus.shape"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(47184, 7)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZEWnr931vSZO"
      },
      "source": [
        "# remove special characters\n",
        "df_corpus['comments_cleaned'] = df_corpus['comments_cleaned'].str.replace('[^a-zA-Z0-9]', ' ')\n",
        "\n",
        "# remove white spaces\n",
        "df_corpus['comments_cleaned'] = df_corpus['comments_cleaned'].str.replace('\\s+', ' ', regex=True)\n",
        "df_corpus['comments_cleaned'] = df_corpus['comments_cleaned'].str.strip()"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w1DNb4RIvSZO"
      },
      "source": [
        "df_corpus['comments_cleaned'].to_csv('meta.tsv', columns=['comments_cleaned'], index= False, header= False)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iX6NhMd5vSZO"
      },
      "source": [
        "The longest comment has 9649 characters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6XEek6ClvSZP",
        "outputId": "8ee048c6-0dd9-48f8-c66d-58d61aacf7da"
      },
      "source": [
        "df_corpus['comments_cleaned'].apply(len).sort_values(ascending=False).head(20)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "13649    9649\n",
              "8742     9572\n",
              "14669    9357\n",
              "39329    8032\n",
              "18645    7502\n",
              "13222    7429\n",
              "4739     7033\n",
              "48888    6823\n",
              "17436    6714\n",
              "26363    6246\n",
              "30068    6076\n",
              "19148    5943\n",
              "9159     5886\n",
              "46668    5758\n",
              "4851     5732\n",
              "15616    5653\n",
              "5239     5653\n",
              "3757     5540\n",
              "29311    5498\n",
              "13054    5313\n",
              "Name: comments_cleaned, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mcv7vIhgvSZP"
      },
      "source": [
        "## Sentence Bert Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9ZboLRN5vSZP"
      },
      "source": [
        "corpus_embeddings = embedder.encode(df_corpus[\"comments_cleaned\"].values.tolist())"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3UcZAIVdvSZP",
        "outputId": "f54636d1-01c8-4a2d-df07-e07973476be0"
      },
      "source": [
        "corpus_embeddings = np.array(corpus_embeddings)\n",
        "corpus_embeddings.shape"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(47184, 768)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MfBQw8jWvSZP"
      },
      "source": [
        "def plot_similarity(labels, features, rotation):\n",
        "  corr = np.inner(features, features)\n",
        "  sns.set(font_scale=1.2)\n",
        "  g = sns.heatmap(\n",
        "      corr,\n",
        "      xticklabels=labels,\n",
        "      yticklabels=labels,\n",
        "      vmin=0,\n",
        "      vmax=1,\n",
        "      cmap=\"YlOrRd\")\n",
        "  g.set_xticklabels(labels, rotation=rotation)\n",
        "  g.set_title(\"Semantic Textual Similarity\")\n",
        "\n",
        "def run_and_plot(messages_):\n",
        "  message_embeddings_ = embed(messages_)\n",
        "  plot_similarity(messages_, corpus_embeddings, 90)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dDMPsKE5vSZQ"
      },
      "source": [
        "## Google Universal Encoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kd-JU_exvSZQ",
        "outputId": "c62ba0a3-81ff-4409-9dad-02d0d19f36c2"
      },
      "source": [
        "from absl import logging\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "import tensorflow_hub as hub\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import os\n",
        "import pandas as pd\n",
        "import re\n",
        "import seaborn as sns\n",
        "\n",
        "module_url = \"https://tfhub.dev/google/universal-sentence-encoder-large/5\" #@param [\"https://tfhub.dev/google/universal-sentence-encoder/4\", \"https://tfhub.dev/google/universal-sentence-encoder-large/5\"]\n",
        "model = hub.load(module_url)\n",
        "print (\"module %s loaded\" % module_url)\n",
        "def embed(input):\n",
        "  return model(input)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:absl:Using /tmp/tfhub_modules to cache modules.\n",
            "INFO:absl:Downloading TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder-large/5'.\n",
            "INFO:absl:Downloaded https://tfhub.dev/google/universal-sentence-encoder-large/5, Total size: 577.10MB\n",
            "INFO:absl:Downloaded TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder-large/5'.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "module https://tfhub.dev/google/universal-sentence-encoder-large/5 loaded\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "img28FjgvSZQ",
        "outputId": "f5bac2a4-0c3c-4392-8a8b-4b58b9f3bfcb"
      },
      "source": [
        "#@title Compute a representation for each message, showing various lengths supported.\n",
        "word = \"Elephant\"\n",
        "sentence = \"I am a sentence for which I would like to get its embedding.\"\n",
        "paragraph = (\n",
        "    \"Universal Sentence Encoder embeddings also support short paragraphs. \"\n",
        "    \"There is no hard limit on how long the paragraph is. Roughly, the longer \"\n",
        "    \"the more 'diluted' the emtbedding will be.\")\n",
        "messages = [word, sentence, paragraph]\n",
        "\n",
        "# Reduce logging output.\n",
        "logging.set_verbosity(logging.ERROR)\n",
        "\n",
        "message_embeddings = embed(messages)\n",
        "\n",
        "for i, message_embedding in enumerate(np.array(message_embeddings).tolist()):\n",
        "  print(\"Message: {}\".format(messages[i]))\n",
        "  print(\"Embedding size: {}\".format(len(message_embedding)))\n",
        "  message_embedding_snippet = \", \".join(\n",
        "      (str(x) for x in message_embedding[:3]))\n",
        "  print(\"Embedding: [{}, ...]\\n\".format(message_embedding_snippet))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Message: Elephant\n",
            "Embedding size: 512\n",
            "Embedding: [-0.03458563610911369, -0.017990142107009888, 0.001980570377781987, ...]\n",
            "\n",
            "Message: I am a sentence for which I would like to get its embedding.\n",
            "Embedding size: 512\n",
            "Embedding: [0.05833395943045616, -0.08185011148452759, 0.06890935450792313, ...]\n",
            "\n",
            "Message: Universal Sentence Encoder embeddings also support short paragraphs. There is no hard limit on how long the paragraph is. Roughly, the longer the more 'diluted' the emtbedding will be.\n",
            "Embedding size: 512\n",
            "Embedding: [-0.018883388489484787, -0.0031109368428587914, -0.033673327416181564, ...]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xFNDvKCSvSZQ"
      },
      "source": [
        "## Semantic Textual Similarity Task Example\n",
        "\n",
        "The embeddings produced by the Universal Sentence Encoder are approximately normalized. The semantic similarity of two sentences can be trivially computed as the inner product of the encodings."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iyhhupo_vSZR"
      },
      "source": [
        "def plot_similarity(labels, features, rotation):\n",
        "  corr = np.inner(features, features)\n",
        "  sns.set(font_scale=1.2)\n",
        "  g = sns.heatmap(\n",
        "      corr,\n",
        "      xticklabels=labels,\n",
        "      yticklabels=labels,\n",
        "      vmin=0,\n",
        "      vmax=1,\n",
        "      cmap=\"YlOrRd\")\n",
        "  g.set_xticklabels(labels, rotation=rotation)\n",
        "  g.set_title(\"Semantic Textual Similarity\")\n",
        "\n",
        "def run_and_plot(messages_):\n",
        "  message_embeddings_ = embed(messages_)\n",
        "  plot_similarity(messages_, message_embeddings_, 90)"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZKYBmWgwvSZR"
      },
      "source": [
        "messages0 = df_corpus[\"comments_cleaned\"].values.tolist()"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OBn-kGVuvSZR",
        "outputId": "9da36e54-7514-42f6-8607-054b9240f342"
      },
      "source": [
        "df_corpus[\"comments_cleaned\"].apply(len).max()"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "9649"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QF4xETc3vSZS"
      },
      "source": [
        "# embeddings_long = embed([df_corpus['comments_cleaned'].loc[617593]])"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YOe8pabYvSZS"
      },
      "source": [
        "# [USComments.csv] Take only the first 2200 characters of each comment. Crashes at 2300+\n",
        "# [Presidential Debate 1] < 2200. Trying 2000 [Nope]. Trying 1800 [Nope].\n",
        "# Trying 1000 [Nope]. Trying 500 [Nope]. Trying 200.\n",
        "max_chars = 200\n",
        "messages = df_corpus[\"comments_cleaned\"].apply(\n",
        "    lambda x: x[:max_chars]\n",
        ").values.tolist()"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pL93dJEI51OD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "44bda6a2-5ea9-4bc4-ff79-af7fa6b12589"
      },
      "source": [
        "print('Embeddings can only be done with about 500 rows at a time')\n",
        "n_chunks = len(messages) // 500\n",
        "message_list = np.array_split(np.array(messages), n_chunks)\n",
        "print(f'The {len(messages)} messages will be divided into {n_chunks} chunks of approximately 500 rows per chunk')"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Embeddings can only be done with about 500 rows at a time\n",
            "The 47184 messages will be divided into 94 chunks of approximately 500 rows per chunk\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oEr9Lh407TV_"
      },
      "source": [
        "for i, message in enumerate(message_list):\n",
        "    if i == 0:\n",
        "        embeddings = embed(message).numpy()\n",
        "    else:\n",
        "        embedding = embed(message).numpy()\n",
        "        embeddings = np.vstack((embeddings, embedding))"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DoqTDzWd8D2n",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1c67c987-dee1-4886-9b91-28c9be88f0f0"
      },
      "source": [
        "print(f'The embeddings numpy array has shape: {embeddings.shape}')"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The embeddings numpy array has shape: (47184, 512)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9X2OiSSj26FO"
      },
      "source": [
        "Saving 47k rows of data will take a long time (the final size is 588MB)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V_QCoFe2vSZS"
      },
      "source": [
        "np.savetxt('vecs0.tsv', embeddings , delimiter=\"\\t\")"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "70st8gsRAh0Q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5a6d5de5-e92e-4dd4-e12d-379c23565765"
      },
      "source": [
        "# Check the final size of the file\n",
        "!ls -lh vecs.tsv"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-rw-r--r-- 1 root root 588M Mar 22 12:22 vecs.tsv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GlBGndw4vSZV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cfd08634-871f-4d2a-b7c5-87dc225798c0"
      },
      "source": [
        "pd.Series(messages).apply(len).max()"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "200"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IH6VTjAVvSZV"
      },
      "source": [
        "### PCA reduction\n",
        "\n",
        "The size of `vecs.tsv` can be reduced further.\n",
        "Using PCA, the number of dimensions of each embedding is reduced from 512 to 50."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gso-q5SPvSZV"
      },
      "source": [
        "from sklearn.decomposition import PCA"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b8e9cd7JvSZV"
      },
      "source": [
        "#scale the data 0-1\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "scaler = MinMaxScaler()\n",
        "rescaled = scaler.fit_transform(embeddings)"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JLfDLcITvSZV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "635900a2-660f-4eee-cea6-027cf4ddb918"
      },
      "source": [
        "pca = PCA().fit(rescaled)\n",
        "plt.plot(np.cumsum(pca.explained_variance_ratio_))\n",
        "plt.xlabel('number of components')\n",
        "plt.ylabel('cumulative explained variance');"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5xcdb3/8ddnW7akbJLd9EYqCS2BkASQJlVU4IIFRESUpoBYr+i9P0W93qtYuYoiIAgqICgizUsvSks2IYX0Td+03ZTdbG/z+f1xzoYhppyU2bM7834+HvOYOd9zZs7nu5mcz5zvOd/v19wdERHJXFlxByAiIvFSIhARyXBKBCIiGU6JQEQkwykRiIhkuJy4A9hfJSUlPmrUqLjDEBHpVmbPnr3F3Ut3t67bJYJRo0ZRVlYWdxgiIt2Kma3Z0zo1DYmIZDglAhGRDKdEICKS4ZQIREQynBKBiEiGS1kiMLN7zKzSzN7Zw3ozs/81s3Izm29mx6YqFhER2bNUnhH8Djh3L+s/AIwLH9cAv05hLCIisgcp60fg7q+a2ai9bHIBcL8H42C/aWbFZjbY3TemKiYRkc7S1p6gsbWdptYETa3t4SNBS3s7re1Oa3sifASv29qdlvA5eV1be4LWhIM7Z0wcyDHDiw95rHF2KBsKrEtargjL/iURmNk1BGcNjBgxolOCE5HMkUg4O5paqW1qo645fDS1UdvcRn3S67qmcLk5WG5qaaeprZ3Gnc8JmlvbaWxtpy1xaOd6MYMBvfPTLhFE5u53AncCTJ06VTPpiMgeJRLOtoYWqmqb2VLXzPaGVmoaWtje0Mr2hhZqwuftDa3UNIZlja3sa44uMyjKy6Fnjxx65udQ1COHwtxs+hflUdA3m/ycbHrkZlOQm01+blb4HLzO3/k6m7ycLHKzjdzsLHKzs8jJsrDs3dc5WUZuTha5WcG22VmGmaXsbxZnIlgPDE9aHhaWiYj8i5a2BJtqmthY00hlbTNVtc1U1YXP4UG/qraZrfUttO/h13jPHjkUF+ZSXJhL38I8hvcrpLggl76FufQpzKN3fg698nPo2SOXoh7ZO1/3zA8O+llZqTsYxynORPA4cIOZPQRMB2p0fUAkM7k7VbXNrK9uZEN1cLDfUN3EhurG4HVNE1vqmv/lV3tOllHaqwelvXowsHc+Rw7ps3O5tFcP+hfl0a8oj+LCPPoU5JKXozvmdydlicDMHgROA0rMrAL4NpAL4O53AE8D5wHlQANwZapiEZH4tbYnqNjeyJqt9azd1sCarcFj7bZguak18Z7tC/OyGdwnnyHFBRw+qDeDi/MZ0qeAwcX5DOydT2nPHvQpyE3bX+mdKZV3DV26j/UOXJ+q/YtI5+v4ZV9eWceKqjrKK+tYuaWe1Vvr2VDd9J4mm/zcLEb0K2Rk/yJOGVfKiP6FDC0uYHCfAoYWF9C7ICel7eLyrm5xsVhEupZEwlm7rYHyyjrKq+pYET6XV9ZR29S2c7uePXIYXVrElOF9uXBy4c4D/8j+hQzo1UMH+i5CiUBE9qq2qZUlm2pZvHEHizcGz0s31dLY2r5zm9JePRhb2pMLJw9lTGkRYwf0YuyAngzsrYN9d6BEICJA0KxTsb2RhRtqWLSx48C/g4rtjTu36VOQy8TBvbhk2nAmDurNmAE9GTugJ30KcmOMXA6WEoFIhqqqbWZ+RTXzKmqYX1HN/IoattW3AJBlcFhJEccML+bSaSOYOLgXEwf3ZlDvfP3CT0NKBCIZoLaplfkVNcyrqGb+uuDAv6GmCQgO+uMG9OKMwwdw9PBijhrahwkDe1GQlx1z1NJZlAhE0oy7s766kdlrtlO2ejtla7azdNMOOm7YGdm/kONG9eMzw/pw9LBijhjSm6IeOhRkMv3ri3Rzbe0JFm+spWzNNsrWbGf26u1s2hH82i/Ky2bKiL7c+P5xHDuyL8cM60NxYV7MEUtXo0Qg0s20tSd4Z8MO3lixlTdWbqVs9TYaWoI7eIb0yef4w/oxdWRfjhvZl8MH9SInW71pZe+UCES6uPaEs3jjuwf+Wau2Udsc3Ks/bkBPLj522M6D/5Digpijle5IiUCki3F3Vm9t4NVlVbxWvoW3Vm2jprEVgNElRXx48hBOGN2fGaP7U9qrR8zRSjpQIhDpAuqa23i9fAuvLq/ilWVVrNsW3Ls/vF8B5xwxkBPHlDBjdH8G9cmPOVJJR0oEIjFIJJxFG3fwyrIqXl1Wxew122lLOIV52Zw4pj/XnDyaU8aXMrJ/UdyhSgZQIhDpJA0tbfxj+RaeX7SZl5ZWsaWuGYBJg3tz1cmjOXV8KceN7KuhkqXTKRGIpNDmHU08v3gzLyyu5J/lW2hpS9ArP4fTJgzgtPGlnDy+hAG91Nwj8VIiEDmE3J3FG2t5fvFmnl+8mfkVNUDQ1v/J6SM5c+IAjj+sH7m6pVO6ECUCkYPk7ryzfgdPLdjI39/ZyJqtDZjBlOHF/Pu5Ezhz4kDGDeipMXqky1IiEDkA7s78ihqeXrCRp9/ZyLptjeRkGSeOLeG6U8dw5sSBurVTug0lApGI3J15FTU8NX8DTy/YxPrq4OB/0tgSbjx9HGdNGkjfIg3fIN1PpERgZiOBce7+vJkVADnuXpva0ES6hlVb6nns7fU8Nnc9a7Y2kJttvG9sCV88cxxnTxpEn0KNxS/d2z4TgZldDVwD9APGAMOAO4AzUhuaSHy21DXz5LwN/HXuBuatq8YMThzTn+tPH8s5RwzSRCySVqKcEVwPTAPeAnD35WY2IKVRicSgoaWN5xZt5q9vr+cfy7fQnnAmDe7NN887nPOPGapevZK2oiSCZndv6bjjwcxyAE9pVCKdxN2Zs3Y7D8+q4Mn5G6hvaWdIn3yuOWU0F04eyoRBveIOUSTloiSCV8zsm0CBmZ0FfB54IrVhiaRWZW0Tf52znofL1rGiqp7CvGw+eNRgPnLcMI4f1Y+sLN3qKZkjSiK4GfgssAC4FngauDuVQYmkQlt7gpeWVvFw2TpeXFJJe8KZOrIvt148hg8ePVizdEnGivLNLwDucfe7AMwsOyxrSGVgIofKppomHpy5lgdnrqWytpmSnj246uTD+Ohxwxk7oGfc4YnELkoieAE4E6gLlwuAZ4ETUxWUyMFyd95YsZXfv7mGZxdtJuHOaeNL+f70kZw2oVRDPIgkiZII8t29Iwng7nVmVpjCmEQOWE1jK4/OqeAPb65hRVU9fQtzuerkw7hs2khG9NfXVmR3oiSCejM71t3nAJjZcUBjasMS2T8rquq455+reHTOehpb25kyopiffuwYzjtqMPm52XGHJ9KlRUkEXwQeMbMNgAGDgI+nNCqRCNydN1Zu5bf/WMULSyrJy8niwslD+NQJozhyaJ+4wxPpNvaZCNx9lpkdDkwIi5a6e2tqwxLZs5a2BE/O38Dd/1jFoo076F+Ux01njOPyE0ZS0lMDvYnsr6j3yx0PjAq3P9bMcPf7UxaVyG7UNLbyhzfXcN/rq6msbWbsgJ784KKjuHDKUDX/iByEKGMN/Z5gjKG5QHtY7IASgXSKqtpm7nltFb9/Yw11zW2cPK6EWz9yNKeOL9UY/yKHQJQzgqnAJHfXsBLSqdZXN3LnKyt4aNY6WtoTnHfUYD5/2hiOGKL2f5FDKUoieIfgAvHGFMciAsDKqjp+/fIK/vr2egAuOnYo1506htGl6vwlkgpREkEJsMjMZgLNHYXufv6+3mhm5wK3AdnA3e7+g13WjwDuA4rDbW5296ejhy/pZGVVHbe9sJzH520gLzuLT84YydWnjGZocUHcoYmktSiJ4JYD+eBwKIrbgbOACmCWmT3u7ouSNvtP4GF3/7WZTSIYx2jUgexPuq912xq47YXlPDqngh452VxzymiuPnm07gAS6SRRbh995QA/expQ7u4rAczsIeACIDkRONA7fN0H2HCA+5JuaEN1I794sZxHytaRlWVcedJhXHfqGM31K9LJotw1NAP4BTARyCNowql39957fSMMBdYlLVcA03fZ5hbgWTO7ESgiGNNodzFcQzBLGiNGjNhXyNLFVdY2cfuL5Tw4cx2O84npI7j+9LEM7K2JX0TiEKVp6JfAJcAjBHcQfQoYf4j2fynwO3f/iZmdAPzezI5090TyRu5+J3AnwNSpU3X3UjdV39zGna+u5K5/rKSlLcFHpw7jhveP0zUAkZhF6lDm7uVmlu3u7cC9ZvY28I19vG09MDxpeVhYluyzwLnhPt4ws3yCi9OVUeKS7qGtPcGfytbxs+eWs6WumQ8eNZivnTOBUSVFcYcmIkRLBA1mlgfMNbNbCW4jjTKG7yxgnJkdRpAALgE+scs2a4EzgN+Z2UQgH6iKGrx0be7O84sr+cHfF7Oiqp7jR/Xlrk8dx5QRfeMOTUSSREkElxNcF7gB+BLBr/yL9/Umd28zsxuAZ8L33+PuC83su0CZuz8OfAW4y8y+RHDh+NPquJYe5q2r5vtPL2bmqm2MLi3izsuP46xJA9UTWKQLsu523J06daqXlZXFHYbsQVVtMz96ZgkPl1VQ0jOPm84czyXHD9dEMCIxM7PZ7j51d+v2eEZgZg+7+8fMbAHBr/X3cPejD2GM0s21tie47/XV3Pb8cpra2rn2lNHc8P6x9MrPjTs0EdmHvTUN3RQ+f6gzApHu65/Lt3DLEwspr6zjlPGlfPvDkxij4SBEuo09JgJ33xj2Dv6du5/eiTFJN7F5RxPfeWIhTy/YxIh+hdz1qamcOXGArgOIdDN7vVjs7u1mljCzPu5e01lBSdfWnnD++NYabv2/pbS2J/jKWeO5+pTRmhNApJuKctdQHbDAzJ4D6jsK3f0LKYtKuqxFG3bwzb8uYO66at43toT/uvBI9QcQ6eaiJIJHw4dksMaWdn7+/DLu/ucqigty+fnHJ3PB5CFqBhJJA1EGnbuvMwKRrmvmqm187c/zWLO1gUuOH87NHzic4sK8uMMSkUMkyqBz44D/ASYR9PwFwN1HpzAu6QIaW9r50TNLuff1VQzvW8iDV8/ghDH94w5LRA6xKE1D9wLfBn4GnA5cSbQhJqQbK1u9ja/9eT6rttRzxQkj+foHDqcwL9LQVCLSzUT5n13g7i+Ymbn7GuAWM5sNfCvFsUkMmlrb+cmzS7n7n6sYWlzAA1dP58QxJXGHJSIpFCURNJtZFrA8HDtoPaDeQmloyaYd3PjA2yyvrOOTM0bwjQ9MpKiHzgJE0l2U/+U3AYXAF4DvETQPXZHKoKRzuTv3v7GG7z+9mD4Fudz/mWmcMr407rBEpJNESQTt7l5H0J/gyhTHI51sW30L//7neTy/uJLTJ5Tyo48eo7mCRTJMlETwEzMbBPwZ+JO7v5PimKSTvF6+hS/+aS7VDa1860OTuPKkUeoXIJKBovQjOD1MBB8DfmNmvQkSwn+lPDpJiUTCuf2lcn76/DJGlxRx75XHc8SQPnGHJSIxiXQbqLtvcvf/Ba4D5qI7hrqtmoZWrr6/jJ88t4zzjxnCEze+T0lAJMNF6VA2Efg4waxkW4E/EcwsJt3Mwg01fO4Pc9hQ3ch3zj+CT50wUk1BIhLpGsE9wEPAOe6+IcXxSIr8eXYF//HXBRQX5vKna0/guJGaN1hEAlGuEZzQGYFIarS2J/jOEwv5w5trmTG6H7+49FhKe+muIBF5l3oLpbHt9S187o+zeXPlNq49ZTRfO2cCOZo7WER2oUSQpsora/nsfWVsrGniZx8/hn+bMizukESki1IiSEMvL63kxgfepkduNg9dM4NjR+h6gIjs2R4TgZk9Afie1rv7+SmJSA7Kfa+v5jtPLOTwQb2564qpDC0uiDskEeni9nZG8OPw+SJgEPCHcPlSYHMqg5L95+7c+sxSfv3yCs6aNJDbLpmsYaNFJJI9Hinc/RUAM/uJu09NWvWEmZWlPDKJrLU9wTceXcCfZ1fwiekj+N4FR5Kdpf4BIhJNlJ+MRWY22t1XApjZYYBmK+8iGlvauf6BOby4pJIvnTmeL5wxVp3ERGS/REkEXwJeNrOVgAEjgWtTGpVEsr2+hc/cN4t566r5/r8dyWXTR8Ydkoh0Q1E6lP1fOG/x4WHREndvTm1Ysi+bapr45G/fYu22Bn512XGce+SguEMSkW4qylhDhcCXgZHufrWZjTOzCe7+ZOrDk91ZX93IJ+56ky21zdz/mWnMGK0J5UXkwEXpZnov0AJ0DDWxHtAQ1DFZu7WBj93xBtvqW/jDVdOVBETkoEVJBGPc/VagFcDdGwiuFUgnW7Wlno/f+Qb1LW08cNUMpqijmIgcAlEuFreYWQFh5zIzGwPoGkEnW1lVxyV3vkl7wnngqhlMGtI77pBEJE1ESQTfBv4PGG5mfwROAj6dyqDkvdZta+Cyu9+iPeE8dM0Mxg3sFXdIIpJGotw19JyZzQFmEDQJ3eTuW1IemQCweUcTl939Fg0t7UoCIpISUcckzge2AzuASWZ2SpQ3mdm5ZrbUzMrN7OY9bPMxM1tkZgvN7IGI8WSErXXNXHb3W2yta+a+z0xj4mA1B4nIoRfl9tEfEkxVuRBIhMUOvLqP92UDtwNnARXALDN73N0XJW0zDvgGcJK7bzezAQdUizRU09jK5b+dScX2Bu67chqThxfHHZKIpKko1wguBCYcQCeyaUB50tAUDwEXAIuStrkauN3dtwO4e+V+7iMtNba0c+W9M1leWcvdVxzPdN0iKiIpFKVpaCWQewCfPRRYl7RcEZYlGw+MN7PXzOxNMzv3APaTVtraE9z44BzmrqvmF5dO4dTxpXGHJCJpLsoZQQMw18xeIOm2UXf/wiHa/zjgNGAY8KqZHeXu1ckbmdk1wDUAI0aMOAS77Zrcnf/3t4U8v7iS7114JOceOTjukEQkA0RJBI+Hj/21HhietDwsLEtWAbzl7q3AKjNbRpAYZiVv5O53AncCTJ06dY+T5XR3v3yxnAdnruXzp43h8hkaQE5EOkeU20fvO8DPngWMC4etXg9cAnxil20eI5jo5l4zKyFoKlp5gPvr1h6dU8FPnlvGRVOG8rVzJsQdjohkkL1NVfmwu3/MzBawmykr3f3ovX2wu7eZ2Q3AM0A2cI+7LzSz7wJl7v54uO5sM1sEtANfc/etB1GfbmnO2u3c/JcFnDimPz+4+GjNJyAincrcd9/SYmaD3X2jme22jcLd16Q0sj2YOnWql5WlzwRpG2saOf+Xr1GYl81jnz+JvkV5cYckImnIzGbvMtvkTnubqnJj+BzLAT8TNLW2c839s2lobuOPV01XEhCRWOzz9lEzm2Fms8yszsxazKzdzHZ0RnDpzN35+l/m886GGm67ZArjNXSEiMQkSj+CXxJc0F0OFABXEfQYloNw3+ur+dvcDXz17AmcOWlg3OGISAaLNNaQu5cD2e7e7u73Ahnf8etgzF1XzfefXsyZEwfwuVPHxB2OiGS4SB3KzCyPoFPZrcBGog9WJ7uobmjh+j/OYUCvfH780WPIytIdQiISrygH9MsJbv+8Aagn6CR2cSqDSlfuzlcfmUdlbRO3X3YsxYW6OCwi8YvSoazjrqFG4DupDSe9/f7NNTy/uJJvf3iSRhMVkS5jbx3KdtuRrMO+OpTJe62oquO/n17MqeNL+fSJo+IOR0Rkp72dEXyo06JIc63tCb78p7nk52bzo4+o57CIdC1761C2syOZmQ0imF/AgVnuvqkTYksbv3yxnHkVNfzqsmMZ0Ds/7nBERN4jSoeyq4CZwEXAR4A3zewzqQ4sXby9dju/fKmci6YM5byjNKy0iHQ9UW4f/RowpWMwODPrD7wO3JPKwNJBQ0sbX354HoN653PLBUfEHY6IyG5FSQRbgdqk5dqwTPbhp88uY9WWeh64ejq98w9kkjcRkdSLkgjKgbfM7G8E1wguAOab2ZcB3P2nKYyv25q3rpp7XlvFZdNHcOKYkrjDERHZoyiJYEX46PC38FmjpO1Ba3uCr/9lPqW9evD1DxwedzgiInsVJRH80N2bkgvMrMTdt6Qopm7vzldXsmRTLXdefpyahESky4syxMRMM5vRsWBmFxNcLJbdWFFVx20vLOe8owZx9hGD4g5HRGSfopwRXAbcY2YvA0OA/sD7UxlUd+XufPPRBeTnZHHL+bpLSES6hyhjDS0ws+8Dvye4Y+gUd69IeWTd0FMLNvLWqm3814VHMqCXOo6JSPewz0RgZr8FxgBHA+OBJ83sF+6uyWmSNLa0899PLWbS4N5cOm1E3OGIiEQW5RrBAuB0d1/l7s8A04FjUxtW93PHKyvYUNPELecfQbbmGBCRbmSficDdfw6MMLMzw6IW4IspjaqbWV/dyB2vrOBDRw9m2mH94g5HRGS/RBlr6Grgz8BvwqJhwGOpDKq7+e+nF2MG3zhvYtyhiIjstyhNQ9cDJwE7ANx9OTAglUF1JzNXbeOp+Ru57tQxDC0uiDscEZH9FiURNLt7S8eCmeWwlwlrMom784O/L2ZQ73yuPUWT0ItI9xQlEbxiZt8ECszsLOAR4InUhtU9vLikkjlrq/nCGeMoyMuOOxwRkQMSJRHcDFQR3D10LfA08J+pDKo7SCScHz2zlFH9C/no1GFxhyMicsCidChLAHeFDwk9MX8DSzbVctslk8nNjpJPRUS6Jh3BDkBre4KfPbeMwwf14sNHD4k7HBGRg6JEcAAenVPB6q0NfO2cCWSp85iIdHORE4GZFaYykO6irT3Br15ewVFD+/D+w3UXrYh0f1E6lJ1oZouAJeHyMWb2q5RH1kU9tWAja7Y2cP3pYzHT2YCIdH9Rzgh+BpxDOE+xu88DTkllUF1VIuH86qUVjBvQk7MnDYw7HBGRQyJS05C7r9ulqD0FsXR5zy/ezNLNtXz+9DG6NiAiaSPKxDTrzOxEwM0sF7gJWJzasLoed+f2l8oZ0a9QdwqJSFqJckZwHcF4Q0OB9cDkcHmfzOxcM1tqZuVmdvNetrvYzNzMpkb53Di8Vr6VeRU1XHfqGHLUb0BE0kiUMwJz98v294PNLBu4HTgLqABmmdnj7r5ol+16EZxlvLW/++hM97y2ipKeeVx83NC4QxEROaSi/LR9zcyeNbPPmlnxfnz2NKDc3VeGg9Y9BFywm+2+B/wQaNqPz+5Uq7bU8+KSSi6bPpIeORpTSETSS5SJacYTjC10BDDHzJ40s09G+OyhQPJF5oqwbCczOxYY7u5P7e2DzOwaMyszs7KqqqoIuz607n9jNbnZxmXTNQWliKSfqHcNzXT3LxP8yt8G3HewOzazLOCnwFci7P9Od5/q7lNLS0sPdtf7pa65jUfKKvjgUYMZ0FsT0otI+onSoay3mV1hZn8HXgc2EiSEfVkPDE9aHhaWdegFHAm8bGargRnA413tgvFfZldQ19zGlScdFncoIiIpEeVi8TyCqSm/6+5v7MdnzwLGmdlhBAngEuATHSvdvQYo6Vg2s5eBr7p72X7sI6XcnT+8uYbJw4s5Zvj+XB4REek+oiSC0e6+3zOSuXubmd0APANkA/e4+0Iz+y5Q5u6P7+9ndrYF62tYXlnH/1x0VNyhiIikzB4TgZn93N2/SNBc8y+JwN3P39eHu/vTBBPZJJd9aw/bnrbPaDvZo3PWk5eTxXlHDY47FBGRlNnbGcHvw+cfd0YgXU1LW4K/zV3P2ZMG0qcgN+5wRERSZo+JwN1nhy8nu/ttyevM7CbglVQGFreXllayvaGVi4/VNJQikt6i3D56xW7KPn2I4+hyHp1TQUnPHpw8rmTfG4uIdGN7u0ZwKcFdPoeZWfKF3V4EfQnS1vb6Fl5cUskVJ4zSuEIikvb2do2go89ACfCTpPJaYH4qg4rbMws30druXDhF4wqJSPrb2zWCNcAa4ITOC6dreGrBRkb2L+SIIb3jDkVEJOWi9CyeYWazzKzOzFrMrN3MdnRGcHHYVt/C6yu28sGjBmsqShHJCFEawH8JXAosBwqAqwiGl05LzyzcRHvC+eDR6jsgIpkh6qBz5UC2u7e7+73AuakNKz5Pzd/IYSVFTBqsZiERyQxRhphoMLM8YK6Z3UpwATktb6WpaWjljZVbufaU0WoWEpGMEeWAfjnBWEE3APUEI4penMqg4vLyskraE86ZkwbGHYqISKfZ5xlBePcQQCPwndSGE68XFldS0jOPycM00qiIZI69dShbAOxx1FF3PzolEcWktT3By0srOeeIQWRlqVlIRDLH3s4IPtRpUXQBZau3s6OpjTMmqllIRDLLvjqUZYwXFm8mLztLYwuJSMbZ5zUCM6vl3SaiPCAXqHf3tLq/8tXlVUwf3Y+iHlFupBIRSR9RLhb36nhtwT2VFxDML5w2Knc0sWxzHR85TkNOi0jm2a/+AB54DDgnRfHE4rUVWwA4cYyahUQk80RpGrooaTELmAo0pSyiGPxz+Vb6FuaqN7GIZKQoDeIfTnrdBqwmaB5KC+7Oa+VbOHFsiW4bFZGMFOUawZWdEUhcVlTVs2lHE+8bq2YhEclMUZqGDgNuBEYlb+/u56curM7zWnlwfUCJQEQyVZSmoceA3wJPAInUhtP5/lm+hRH9ChnerzDuUEREYhElETS5+/+mPJIYtLUneHPlVj509JC4QxERiU2URHCbmX0beBZo7ih09zkpi6qTLNlUS21TGzNG94s7FBGR2ERJBEcRDEX9ft5tGvJwuVubu64agGNH9I05EhGR+ERJBB8FRrt7S6qD6Wxz11XTvyiPYX0L4g5FRCQ2UXoWvwOk5QD9c9dVM3l4sWYjE5GMFuWMoBhYYmazeO81gm59++iOplZWVNVxwTG6UCwimS1KIvh2yqOIwfx1NbjD5BFpebIjIhJZlJ7Fr3RGIJ1t7rrtABytaSlFJMNl7HwEc9dVM6a0iD4FuXGHIiISq4ydj2BeRY1mIxMRIUPnI9hW30JVbTMTB3XrkxoRkUMipfMRmNm5wG1ANnC3u/9gl/VfBq4iGN66CvhMZ8yVvHRTLQATBvXax5YiIukvZfMRmFk2cDtwFlABzDKzx919UdJmbwNT3b3BzD4H3Ap8PGLsB2zZZiUCEZEOqZyPYBpQ7u4rAczsIYIEsjMRuPtLSdu/CXzyAPe1X5ZsqqVPQaUZsmMAAAsESURBVC4DevXojN2JiHRp+7xGYGb3mVlx0nJfM7snwmcPBdYlLVeEZXvyWeDve4jhGjMrM7OyqqqqCLveu2Wba5kwqJd6FIuIEO1i8dHuXt2x4O7bgSmHMggz+yTBtYcf7W69u9/p7lPdfWppaelB7cvdWbaplgkD1SwkIgLREkGWme0cntPM+hHt2sJ6YHjS8rCw7D3M7EzgP4Dz3b151/WH2qYdTdQ2tzFuYM9U70pEpFuIckD/CfCGmT0SLn8U+H6E980CxoVTXa4HLgE+kbyBmU0BfgOc6+6VkaM+CKu3NABwWElRZ+xORKTLi3Kx+H4zK+Pd+Qcu2uXOnz29r83MbgCeIbh99B53X2hm3wXK3P1xgqagnsAjYXv92lQPZrd6az0Ao/orEYiIQLQzAsID/z4P/rt539PA07uUfSvp9Zn7+5kHa/XWevKysxhSrDkIRERgP3sWp4PVW+oZ3q+A7CzdMSQiAhmYCNZsbVCzkIhIkoxKBImEs3prPaN0oVhEZKeMSgSVtc00tSYY1b8w7lBERLqMjEoEHXcMjVTTkIjIThmVCDbvCAZNHVKcH3MkIiJdR0YlgsodQcfl0l5KBCIiHTIrEdQ20SMni975kbpPiIhkhAxLBM0M6N1Do46KiCTJrESwo5kBahYSEXmPzEoEtU2ajEZEZBcZlQi2N7TSrygv7jBERLqUjEkEiYRT3dBC30IlAhGRZBmTCGqb2kg4FBfmxh2KiEiXkjGJoLqxBUBnBCIiu8iYRLC9oRWAvkU6IxARSZZBiSA4I+hToDMCEZFkGZMIajrOCHSNQETkPTImEXScEegagYjIe2VMIhhaXMDZkwbSu0BnBCIiyTJm9LWzjxjE2UcMijsMEZEuJ2POCEREZPeUCEREMpwSgYhIhlMiEBHJcEoEIiIZTolARCTDKRGIiGQ4JQIRkQxn7h53DPvFzKqANQf49hJgyyEMp6vLpPpmUl1B9U1nqarrSHcv3d2KbpcIDoaZlbn71Ljj6CyZVN9MqiuovuksjrqqaUhEJMMpEYiIZLhMSwR3xh1AJ8uk+mZSXUH1TWedXteMukYgIiL/KtPOCEREZBdKBCIiGS5jEoGZnWtmS82s3MxujjueQ8HM7jGzSjN7J6msn5k9Z2bLw+e+YbmZ2f+G9Z9vZsfGF/n+M7PhZvaSmS0ys4VmdlNYnnb1NbN8M5tpZvPCun4nLD/MzN4K6/QnM8sLy3uEy+Xh+lFxxn+gzCzbzN42syfD5bSsr5mtNrMFZjbXzMrCsli/xxmRCMwsG7gd+AAwCbjUzCbFG9Uh8Tvg3F3KbgZecPdxwAvhMgR1Hxc+rgF+3UkxHiptwFfcfRIwA7g+/DdMx/o2A+9392OAycC5ZjYD+CHwM3cfC2wHPhtu/1lge1j+s3C77ugmYHHScjrX93R3n5zUXyDe77G7p/0DOAF4Jmn5G8A34o7rENVtFPBO0vJSYHD4ejCwNHz9G+DS3W3XHR/A34Cz0r2+QCEwB5hO0Ns0Jyzf+Z0GngFOCF/nhNtZ3LHvZz2HERwA3w88CVi61hdYDZTsUhbr9zgjzgiAocC6pOWKsCwdDXT3jeHrTcDA8HXa/A3CpoApwFukaX3DZpK5QCXwHLACqHb3tnCT5PrsrGu4vgbo37kRH7SfA/8OJMLl/qRvfR141sxmm9k1YVms3+OMmbw+E7m7m1la3R9sZj2BvwBfdPcdZrZzXTrV193bgclmVgz8FTg85pBSxsw+BFS6+2wzOy3ueDrB+9x9vZkNAJ4zsyXJK+P4HmfKGcF6YHjS8rCwLB1tNrPBAOFzZVje7f8GZpZLkAT+6O6PhsVpW18Ad68GXiJoGik2s44fb8n12VnXcH0fYGsnh3owTgLON7PVwEMEzUO3kab1dff14XMlQZKfRszf40xJBLOAceFdCHnAJcDjMceUKo8DV4SvryBoS+8o/1R4F8IMoCbpVLTLs+Cn/2+Bxe7+06RVaVdfMysNzwQwswKCayGLCRLCR8LNdq1rx9/gI8CLHjYodwfu/g13H+buowj+b77o7peRhvU1syIz69XxGjgbeIe4v8dxXzjpxAs05wHLCNpa/yPueA5RnR4ENgKtBG2HnyVoK30BWA48D/QLtzWCO6dWAAuAqXHHv591fR9B2+p8YG74OC8d6wscDbwd1vUd4Fth+WhgJlAOPAL0CMvzw+XycP3ouOtwEHU/DXgyXesb1mle+FjYcSyK+3usISZERDJcpjQNiYjIHigRiIhkOCUCEZEMp0QgIpLhlAhERDKcEoF0a2b2spmlfKJvM/uCmS02sz+mel9xMrNiM/t83HFI51IikIyV1Gs1is8DZ3nQ0SmdFRPUVTKIEoGknJmNCn9N3xWOr/9s2GP2Pb/ozawkHGYAM/u0mT0Wjs2+2sxuMLMvh+PVv2lm/ZJ2cXk4tvs7ZjYtfH+RBfM1zAzfc0HS5z5uZi8SdODZNdYvh5/zjpl9MSy7g6Aj0N/N7Eu7bJ9tZj8Ot59vZjeG5WeE+10QxtEjLF9tZv/TMRa9mR1rZs+Y2Qozuy7c5jQze9XMnrJgDo07zCwrXHdp+JnvmNkPk+KoM7PvWzCHwZtmNjAsLzWzv5jZrPBxUlh+SxjXy2a20sy+EH7UD4AxYXw/MrPBYSwdf9+TD/iLIF1X3D3t9Ej/B8FQ2W3A5HD5YeCT4euXCXtLAiXA6vD1pwl6jvYCSglGmLwuXPczgkHnOt5/V/j6FMIhuYH/TtpHMUGv8qLwcysIe27uEudxBL03i4CeBD0/p4TrVrPL0MFh+eeAP/PucMn9CHq+rgPGh2X3J8W7GvhcUj3mJ9Vxc1h+GtBEkHyyCUYf/QgwBFgbbpsDvAhcGL7HgQ+Hr28F/jN8/QDBIGcAIwiG6AC4BXgd6BH+3bcCufzrsOZf4d3er9lAr7i/T3oc+odGH5XOssrd54avZxMccPblJXevBWrNrAZ4IixfQDAMQ4cHAdz9VTPrHY7TczbBQGZfDbfJJzgQAjzn7tt2s7/3AX9193oAM3sUOJlguIc9ORO4w8Phkt19m5kdE9Z3WbjNfcD1BEMtw7vjXC0AeibVsbljjCFgpruvDON4MIytFXjZ3avC8j8SJL/HgBaCcfwh+PuelRTfJHt3lNbeFozgCvCUuzcDzWZWybtDHyebBdxjwYB/jyX9G0oaUSKQztKc9LodKAhft/FuE2X+Xt6TSFpO8N7v7q7jpDjBGC0Xu/vS5BVmNh2o36/ID73keuxax4567a5Oe9Pq7h3btCd9ThYww92bkjcOE8Ou/yb/cjwIk+spwAeB35nZT939/n3EIt2MrhFI3FYTNMnAuyNN7q+PA5jZ+whGZ6whmMXqRguPeGY2JcLn/AO40MwKw5Eh/y0s25vngGs7LjyH1y6WAqPMbGy4zeXAK/tZp2kWjJabRVC/fxIMsHZqeC0lG7g0wuc+C9zYsWBmk/exfS1BU1XH9iMJmqzuAu4Gus3czxKdEoHE7cfA58zsbYK26gPRFL7/Dt6d1/Z7BG3e881sYbi8V+4+h2Ae6JkEs5/d7e57axaC4OC4NtzPPOAT4a/vK4FHzGwBwS/9O/azTrOAXxIMP72KoMlqI8Fcti8RjF45293/tuePAOALwNTwQvYi4Lq9bezuW4HXwgvDPyK4XjEv/Pt+nGCeAEkzGn1UpIuxYJaur7r7h+KORTKDzghERDKczghERDKczghERDKcEoGISIZTIhARyXBKBCIiGU6JQEQkw/1/TKdkAey+/pwAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n4qYCfpbvSZV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8485458e-8b56-4f51-f080-7722c3bca441"
      },
      "source": [
        "# need around 200 components to describe 100% of variance\n",
        "pca = PCA(n_components = 50)\n",
        "reduced_embeds = pca.fit_transform(rescaled)\n",
        "print(\"Original shape:   \", rescaled.shape)\n",
        "print(\"Transformed shape:\", reduced_embeds.shape)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original shape:    (47184, 512)\n",
            "Transformed shape: (47184, 50)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMYogewKvSZW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7f4ef568-1242-4316-cc65-7ef801a584d6"
      },
      "source": [
        "pca = PCA(n_components = 50)\n",
        "reduced_embeds = pca.fit_transform(rescaled)\n",
        "reduced_embeds.shape"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(47184, 50)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kVexQw8e9UYT"
      },
      "source": [
        "np.savetxt('vecs.tsv', reduced_embeds , delimiter=\"\\t\")"
      ],
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-H6f_9ZR-Z3w"
      },
      "source": [
        "After dimension reduction using PCA, the file size has been reduced from 588 MB to 58 MB."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ubaCiKEK9bNe",
        "outputId": "b43aab1b-4d6f-419f-caaa-36ac8d2a8878"
      },
      "source": [
        "!ls -lh vecs*.tsv"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-rw-r--r-- 1 root root 58M Mar 22 12:42 vecs2.tsv\n",
            "-rw-r--r-- 1 root root 58M Mar 22 12:46 vecs.tsv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wwHbLUzpvSZW"
      },
      "source": [
        "## Run HDBScan to find the clusters\n",
        "\n",
        "Note: the next code block takes a long time"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hVxDNTW4vSZX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03cdeeb7-e885-4f1b-e0bb-ac95f5705073"
      },
      "source": [
        "import hdbscan\n",
        "\n",
        "clusterer = hdbscan.HDBSCAN()\n",
        "clusterer.fit(reduced_embeds)\n",
        "clusterer.labels_"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ -1,  -1,  -1, ...,  -1,  -1, 207])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MdzdxO4gvSZX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b675610a-9555-443f-c7ba-13f2e880705e"
      },
      "source": [
        "clusterer.labels_.max()"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "334"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GuyjHVj7vSZX"
      },
      "source": [
        "df_corpus['hdb_labels'] = clusterer.labels_"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ItoZ2c79vSZX"
      },
      "source": [
        "# df_corpus.to_csv('meta_lab.tsv', columns=['comments_cleaned','hdb_labels'], index= False, header= True, sep='\\t')\n",
        "df_corpus.to_csv('meta_lab.tsv', columns=['comment_text','hdb_labels'], index= False, header= True, sep='\\t')"
      ],
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iGK0xEcJvSZY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2b123ec2-827b-43b4-b159-399aa78aa7aa"
      },
      "source": [
        "!head -5 meta.tsv"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "25 28 trump did you hear anything biden no what was that sound\n",
            "53 39 trump the idiot your the big man i dont know if you are but your the big man which is it trump grammer helps\n",
            "trump is the president of the usa no biden\n",
            "driving home from school today because that wednesday i m going to schools for the day so i m not going out of the house to get wills house\n",
            "trump won election the swamp rats in washington did everything they could to cheat in order to have biden win congrats you now have the power and this country and all we can do is sit around and watch you waste our money change the constitution do whatever they want and clearly i don t see our country strong and proud this election was a disgrace 75 million people voted for president trump and the few swing states should be ashamed of themselves for being pushed around to be bullied into being dishonest no one on trump side believes for one second that sleepy joe won he is not my president i am sad angry frustrated and upset so pelosi and shumner shame on you you have not done 1 thing for the american people you are the reason we all want term limits because you don t deserve to be in government do us all a favor retire\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LVw6ZMisvSZY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d6aa5be4-3ab9-45a2-f663-7f34311769cc"
      },
      "source": [
        "!head -5 meta_lab.tsv"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "comments_cleaned\thdb_labels\n",
            "25 28 trump did you hear anything biden no what was that sound\t-1\n",
            "53 39 trump the idiot your the big man i dont know if you are but your the big man which is it trump grammer helps\t-1\n",
            "trump is the president of the usa no biden\t-1\n",
            "driving home from school today because that wednesday i m going to schools for the day so i m not going out of the house to get wills house\t-1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hcb3V6wK5ahi",
        "outputId": "9e6b31d2-8f40-48c6-ee90-1ceccc5ef3c3"
      },
      "source": [
        "!ls -lh *.tsv"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-rw-r--r-- 1 root root 6.2M Mar 22 13:10 meta_lab.tsv\n",
            "-rw-r--r-- 1 root root 6.0M Mar 22 12:15 meta.tsv\n",
            "-rw-r--r-- 1 root root  58M Mar 22 12:42 vecs2.tsv\n",
            "-rw-r--r-- 1 root root  58M Mar 22 12:46 vecs.tsv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mh1NbTfbvSZY"
      },
      "source": [
        "Save`vecs.tsv` and `meta_lab.tsv` and navigate to http://projector.tensorflow.org/. Press \"Load\" and use `vecs.tsv` for the first file, and `meta_lab.tsv` for the second file. Downloading files from Google Colab can take a while. For faster downloading, `vecs.tsv` has been compressed as `vecs.tar.bz2` (file reduction from 58 MB to 20 MB)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lV4wksXQ-_TT"
      },
      "source": [
        "!tar -cjf vecs.tar.bz2 vecs.tsv\n",
        "!ls -lh vecs.tar.bz2"
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8HLvTekSvSZY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "outputId": "4ac40ea9-ed84-47ee-eb49-be62939b6e91"
      },
      "source": [
        "from google.colab import files\n",
        "\n",
        "files.download('vecs.tar.bz2')\n",
        "files.download('meta_lab.tsv')"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_315bd419-e124-41c6-97e2-c9345c174141\", \"vecs.tar.bz2\", 20554839)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_7b5e6a1b-a88e-4f16-8906-19af9f2335a2\", \"meta_lab.tsv\", 6401984)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}