LAION/collect_results_imagenet_testbed.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5540b231",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import pandas as pd\n",
    "import os\n",
    "import json\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "import sys\n",
    "sys.path.append('./imagenet-testbed/')\n",
    "sys.path.append('./imagenet-testbed/src/')\n",
    "from registry import registry\n",
    "registry.load_full_registry()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "undefined-firewall",
   "metadata": {},
   "source": [
    "# Arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "09bb87eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_base_dir = './imagenet-testbed/outputs'\n",
    "eval_settings = ['val',\n",
    "                 'val_subsampled_class_1_8',\n",
    "                 'imagenetv2-matched-frequency',\n",
    "                 'imagenet-sketch',\n",
    "                 'ytbb-robust',\n",
    "                 'imagenet-vid-robust',\n",
    "                 'objectnet-1.0-beta',\n",
    "                 'imagenet-a', \n",
    "                 'imagenet-r']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "classified-television",
   "metadata": {},
   "source": [
    "# Collect Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6d3817e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['clip_vit_pretrained', 'clip_vit_finetuned_base', 'clip_vit_finetuned_ent']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_models = [m for m in registry.model_names() if 'clip' in m]\n",
    "eval_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f02c27df",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_dict = {}\n",
    "\n",
    "for model in eval_models:\n",
    "    if not os.path.exists(os.path.join(os.path.join(result_base_dir, model), 'val/metrics.json')):\n",
    "        print('Not exist: ', model)\n",
    "        continue\n",
    "    result_dict[model] = {}\n",
    "    sub_result_dict = result_dict[model] \n",
    "    \n",
    "    for setting in eval_settings:\n",
    "        result_dir = os.path.join(os.path.join(result_base_dir, model), setting)\n",
    "        result_path = os.path.join(result_dir, 'metrics.json')\n",
    "        \n",
    "        with open(result_path) as json_file:\n",
    "            eval_result_dict = json.load(json_file)\n",
    "        \n",
    "        if setting in ['ytbb-robust', 'imagenet-vid-robust']:\n",
    "            result = (eval_result_dict['pm0'] + eval_result_dict['pm10']) / 2 * 100\n",
    "        else:\n",
    "            result = eval_result_dict['top1']\n",
    "        sub_result_dict[setting] = result\n",
    "    \n",
    "    sub_result_dict['average'] = np.mean([sub_result_dict[s] for s in eval_settings \n",
    "                                           if s not in ['val', 'val_subsampled_class_1_8']])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "raised-quantum",
   "metadata": {},
   "source": [
    "Finetuning CLIP with entropy bottleneck leads to better robustness than finetuning without on natural distribution shift datasets. Note that both fientuned models are worse than pretrained CLIP, probably due to the quality of the LAION-400M dataset.\n",
    "\n",
    "The results could be slightly different from those in the paper, due to code cleaning and randomness."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "lesser-corruption",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>val</th>\n",
       "      <th>val_subsampled_class_1_8</th>\n",
       "      <th>imagenetv2-matched-frequency</th>\n",
       "      <th>imagenet-sketch</th>\n",
       "      <th>ytbb-robust</th>\n",
       "      <th>imagenet-vid-robust</th>\n",
       "      <th>objectnet-1.0-beta</th>\n",
       "      <th>imagenet-a</th>\n",
       "      <th>imagenet-r</th>\n",
       "      <th>average</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>clip_vit_pretrained</th>\n",
       "      <td>75.4</td>\n",
       "      <td>90.6</td>\n",
       "      <td>64.2</td>\n",
       "      <td>41.1</td>\n",
       "      <td>58.3</td>\n",
       "      <td>71.3</td>\n",
       "      <td>42.8</td>\n",
       "      <td>27.7</td>\n",
       "      <td>62.9</td>\n",
       "      <td>52.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clip_vit_finetuned_base</th>\n",
       "      <td>73.8</td>\n",
       "      <td>90.0</td>\n",
       "      <td>62.1</td>\n",
       "      <td>37.0</td>\n",
       "      <td>56.9</td>\n",
       "      <td>68.8</td>\n",
       "      <td>41.3</td>\n",
       "      <td>26.0</td>\n",
       "      <td>58.1</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clip_vit_finetuned_ent</th>\n",
       "      <td>74.0</td>\n",
       "      <td>90.4</td>\n",
       "      <td>62.6</td>\n",
       "      <td>39.0</td>\n",
       "      <td>58.9</td>\n",
       "      <td>69.9</td>\n",
       "      <td>41.9</td>\n",
       "      <td>26.2</td>\n",
       "      <td>60.9</td>\n",
       "      <td>51.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          val  val_subsampled_class_1_8  \\\n",
       "clip_vit_pretrained      75.4                      90.6   \n",
       "clip_vit_finetuned_base  73.8                      90.0   \n",
       "clip_vit_finetuned_ent   74.0                      90.4   \n",
       "\n",
       "                         imagenetv2-matched-frequency  imagenet-sketch  \\\n",
       "clip_vit_pretrained                              64.2             41.1   \n",
       "clip_vit_finetuned_base                          62.1             37.0   \n",
       "clip_vit_finetuned_ent                           62.6             39.0   \n",
       "\n",
       "                         ytbb-robust  imagenet-vid-robust  objectnet-1.0-beta  \\\n",
       "clip_vit_pretrained             58.3                 71.3                42.8   \n",
       "clip_vit_finetuned_base         56.9                 68.8                41.3   \n",
       "clip_vit_finetuned_ent          58.9                 69.9                41.9   \n",
       "\n",
       "                         imagenet-a  imagenet-r  average  \n",
       "clip_vit_pretrained            27.7        62.9     52.6  \n",
       "clip_vit_finetuned_base        26.0        58.1     50.0  \n",
       "clip_vit_finetuned_ent         26.2        60.9     51.3  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option(\"display.precision\", 1)\n",
    "pd.DataFrame.from_dict(result_dict, orient='index')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "optdom",
   "language": "python",
   "name": "optdom"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}