DomainBed/collect_clip_results_per_setup.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "third-governor",
   "metadata": {},
   "source": [
    "Here we collect and analyze the individual results of each setup per left-out test domains. These results can be compared to those in Appendix B of [DomainBed paper](https://arxiv.org/pdf/2007.01434.pdf)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "791508b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import json\n",
    "import os\n",
    "import glob\n",
    "import sys\n",
    "from pprint import pprint\n",
    "from matplotlib import pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "from domainbed.lib import misc, reporting\n",
    "from domainbed import datasets\n",
    "from domainbed import algorithms\n",
    "from domainbed.lib.query import Q\n",
    "from domainbed.model_selection import OracleSelectionMethod"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad41f226",
   "metadata": {},
   "source": [
    "# Arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "88a64e37",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_output_dir=\"./checkpoints\"\n",
    "dataset_all = ['PACS', 'VLCS', 'OfficeHome', 'TerraIncognita', 'DomainNet']\n",
    "algorithm_all = {'CLIPPretrained': 'CLIPPretrained',\n",
    "                 'CLIPBase': 'SupCLIPBottleneckBase',\n",
    "                 'CLIPCondCAD': 'SupCLIPBottleneckCondCAD'}\n",
    "\n",
    "select_method = OracleSelectionMethod"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f77dfeec",
   "metadata": {},
   "source": [
    "# Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d5339eed",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pretty(d, indent=0):\n",
    "    for key, value in d.items():\n",
    "        print('\\t' * indent + str(key))\n",
    "        if isinstance(value, dict):\n",
    "             pretty(value, indent+1)\n",
    "        else:\n",
    "             print('\\t' * (indent+1) + str(value))\n",
    "\n",
    "def get_record(output_dir):\n",
    "    print(\"Loading records from:\", output_dir)\n",
    "    records = reporting.load_records(output_dir)\n",
    "    print(\"Total records:\", len(records))\n",
    "\n",
    "    return records\n",
    "\n",
    "def get_results_per_domain(out_dir, selection_method, num_envs, env_names=None):\n",
    "    \"\"\"Given all records, get averaged results of each setup for each test domain.\"\"\"\n",
    "    records = get_record(out_dir)\n",
    "    \n",
    "    grouped_records = reporting.get_grouped_records(records,\n",
    "                                                    group_test_envs=True).map(\n",
    "        lambda group:\n",
    "        {**group, \"sweep_acc\": selection_method.sweep_acc(group[\"records\"], return_extra=True)}\n",
    "        ).filter(lambda g: g[\"sweep_acc\"] is not None)\n",
    "\n",
    "    # read algorithm names and sort (predefined order)\n",
    "    alg_names = Q(records).select(\"args.algorithm\").unique()\n",
    "    assert len(alg_names) == 1\n",
    "    algorithm = alg_names[0]\n",
    "\n",
    "    # read dataset names and sort (lexicographic order)\n",
    "    dataset_names = Q(records).select(\"args.dataset\").unique().sorted()\n",
    "    assert len(dataset_names) == 1\n",
    "    dataset = dataset_names[0]\n",
    "    \n",
    "    results = {}\n",
    "    for test_env in range(num_envs):\n",
    "        trial_averages = (grouped_records\n",
    "                              .filter_equals(\"algorithm, dataset, test_env\", (algorithm, dataset, test_env))\n",
    "                              .group(\"trial_seed\")\n",
    "                              .map(lambda trial_seed, group:\n",
    "        #                            group.select(\"sweep_acc\").mean()\n",
    "                                   tuple(map(lambda y: sum(y) / float(len(y)), zip(*group.select(\"sweep_acc\"))))\n",
    "                                   )\n",
    "                              )\n",
    "\n",
    "        tgt_all, src_all, tgt_in_all = zip(*trial_averages) \n",
    "        tgt_mean, src_mean, tgt_in_mean = 100 * np.mean(list(tgt_all)), 100 * np.mean(list(src_all)), 100 * np.mean(list(tgt_in_all))\n",
    "        tgt_std, src_std, tgt_in_std = 100 * np.std(list(tgt_all)), 100 * np.std(list(src_all)), 100 * np.std(list(tgt_in_all))\n",
    "\n",
    "        if env_names is not None:\n",
    "            result_key = env_names[test_env]\n",
    "        else:\n",
    "            result_key = f'env_{test_env}'\n",
    "        \n",
    "        results.update({result_key: '{:.1f} +/- {:.1f}'.format(tgt_mean, tgt_std)})\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0de790a1",
   "metadata": {},
   "source": [
    "# Collect Results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "suitable-sullivan",
   "metadata": {},
   "source": [
    "Collect the result dict for each setup:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "british-overall",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_result(setup):\n",
    "    result_dict = {}\n",
    "    for dataset in dataset_all:\n",
    "        result_dict[dataset] = {}\n",
    "        sub_result_dict = result_dict[dataset]\n",
    "        basedir = f'{base_output_dir}/{dataset}/{setup}'\n",
    "        \n",
    "        env_names = datasets.get_dataset_class(dataset).ENVIRONMENTS\n",
    "        num_envs = len(env_names)\n",
    "        for alg_name, alg_name_long in algorithm_all.items():\n",
    "            if alg_name in ['CLIPPretrained', 'CLIPBase']:\n",
    "                sub_result_dict[alg_name] = {}\n",
    "                subsub_result_dict = sub_result_dict[alg_name]\n",
    "\n",
    "                output_dir = os.path.join(basedir, f'{alg_name_long}/base')\n",
    "                subsub_result_dict.update(get_results_per_domain(output_dir, select_method, num_envs, env_names))\n",
    "            else:\n",
    "                lambda_str_array = list(map(lambda s: s.split('_')[-1], glob.glob(os.path.join(basedir, f'{alg_name_long}/*'))))\n",
    "                lambda_str_array = sorted(lambda_str_array, key=lambda r: float(r))\n",
    "                lambda_val_array = np.array(list(map(lambda s: float(s), lambda_str_array)))\n",
    "\n",
    "                for lambda_str in lambda_str_array:\n",
    "                    sub_result_dict[alg_name + '_lambda_{}'.format(lambda_str)] = {}\n",
    "                    subsub_result_dict = sub_result_dict[alg_name + '_lambda_{}'.format(lambda_str)]\n",
    "                    output_dir = os.path.join(basedir, f'{alg_name_long}/lambda_{lambda_str}')\n",
    "\n",
    "                    subsub_result_dict.update(get_results_per_domain(output_dir, select_method, num_envs, env_names))\n",
    "    return result_dict\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "molecular-manual",
   "metadata": {},
   "source": [
    "Result with CLIP-RN50 (CLIP S in our paper):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "eee3a480",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading records from: ./checkpoints/PACS/clip_resnet/CLIPPretrained/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 20\n",
      "Loading records from: ./checkpoints/PACS/clip_resnet/SupCLIPBottleneckBase/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 2000\n",
      "Loading records from: ./checkpoints/PACS/clip_resnet/SupCLIPBottleneckCondCAD/lambda_1e-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 2000\n",
      "Loading records from: ./checkpoints/VLCS/clip_resnet/CLIPPretrained/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 20\n",
      "Loading records from: ./checkpoints/VLCS/clip_resnet/SupCLIPBottleneckBase/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 2000\n",
      "Loading records from: ./checkpoints/VLCS/clip_resnet/SupCLIPBottleneckCondCAD/lambda_1e-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 2000\n",
      "Loading records from: ./checkpoints/OfficeHome/clip_resnet/CLIPPretrained/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 20\n",
      "Loading records from: ./checkpoints/OfficeHome/clip_resnet/SupCLIPBottleneckBase/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 200\n",
      "Loading records from: ./checkpoints/OfficeHome/clip_resnet/SupCLIPBottleneckCondCAD/lambda_1e-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 200\n",
      "Loading records from: ./checkpoints/TerraIncognita/clip_resnet/CLIPPretrained/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 20\n",
      "Loading records from: ./checkpoints/TerraIncognita/clip_resnet/SupCLIPBottleneckBase/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 2181\n",
      "Loading records from: ./checkpoints/TerraIncognita/clip_resnet/SupCLIPBottleneckCondCAD/lambda_1e-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 2122\n",
      "Loading records from: ./checkpoints/DomainNet/clip_resnet/CLIPPretrained/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 30\n",
      "Loading records from: ./checkpoints/DomainNet/clip_resnet/SupCLIPBottleneckBase/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 300\n",
      "Loading records from: ./checkpoints/DomainNet/clip_resnet/SupCLIPBottleneckCondCAD/lambda_1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 300\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "resnet_result_dict = get_result('clip_resnet')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "attractive-insertion",
   "metadata": {},
   "source": [
    "Result with CLIP-ViT-B/32 (CLIP L in our paper):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b736ed34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading records from: ./checkpoints/PACS/clip_vit/CLIPPretrained/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 20\n",
      "Loading records from: ./checkpoints/PACS/clip_vit/SupCLIPBottleneckBase/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 2000\n",
      "Loading records from: ./checkpoints/PACS/clip_vit/SupCLIPBottleneckCondCAD/lambda_1e-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 2000\n",
      "Loading records from: ./checkpoints/VLCS/clip_vit/CLIPPretrained/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 20\n",
      "Loading records from: ./checkpoints/VLCS/clip_vit/SupCLIPBottleneckBase/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 2000\n",
      "Loading records from: ./checkpoints/VLCS/clip_vit/SupCLIPBottleneckCondCAD/lambda_1e-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 2000\n",
      "Loading records from: ./checkpoints/OfficeHome/clip_vit/CLIPPretrained/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 20\n",
      "Loading records from: ./checkpoints/OfficeHome/clip_vit/SupCLIPBottleneckBase/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 200\n",
      "Loading records from: ./checkpoints/OfficeHome/clip_vit/SupCLIPBottleneckCondCAD/lambda_1e-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 200\n",
      "Loading records from: ./checkpoints/TerraIncognita/clip_vit/CLIPPretrained/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 20\n",
      "Loading records from: ./checkpoints/TerraIncognita/clip_vit/SupCLIPBottleneckBase/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 2015\n",
      "Loading records from: ./checkpoints/TerraIncognita/clip_vit/SupCLIPBottleneckCondCAD/lambda_1e-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 2008\n",
      "Loading records from: ./checkpoints/DomainNet/clip_vit/CLIPPretrained/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 30\n",
      "Loading records from: ./checkpoints/DomainNet/clip_vit/SupCLIPBottleneckBase/base\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 300\n",
      "Loading records from: ./checkpoints/DomainNet/clip_vit/SupCLIPBottleneckCondCAD/lambda_1e-1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records: 300\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "vit_result_dict = get_result('clip_vit')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3f78ece",
   "metadata": {},
   "source": [
    "# Plot Results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "short-schema",
   "metadata": {},
   "source": [
    "We plot more fine-grained results broken up to each left-out test domain, which can be compared to those in Appendix B of [DomainBed paper](https://arxiv.org/pdf/2007.01434.pdf).\n",
    "\n",
    "The improvement of CLIP methods over DomainBed baselines varies depending on specific dataset and left-out test domain. Finetuning with bottlenecks (CLIPCondCAD) leads to consistent improvement over the other two CLIP baselines for nearly all setups."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "367072c3",
   "metadata": {},
   "source": [
    "# DomainNet"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "developed-terrorist",
   "metadata": {},
   "source": [
    "CAD leads to significant improvments on DomainNet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "37ddb89b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clip</th>\n",
       "      <th>info</th>\n",
       "      <th>paint</th>\n",
       "      <th>quick</th>\n",
       "      <th>real</th>\n",
       "      <th>sketch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CLIPPretrained</th>\n",
       "      <td>60.4 +/- 0.1</td>\n",
       "      <td>38.5 +/- 0.3</td>\n",
       "      <td>54.7 +/- 0.2</td>\n",
       "      <td>8.2 +/- 0.1</td>\n",
       "      <td>72.8 +/- 0.1</td>\n",
       "      <td>51.1 +/- 0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPBase</th>\n",
       "      <td>60.7 +/- 0.4</td>\n",
       "      <td>35.3 +/- 0.3</td>\n",
       "      <td>53.2 +/- 0.3</td>\n",
       "      <td>9.6 +/- 0.2</td>\n",
       "      <td>71.4 +/- 0.2</td>\n",
       "      <td>50.1 +/- 0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPCondCAD_lambda_1</th>\n",
       "      <td>61.8 +/- 0.5</td>\n",
       "      <td>38.3 +/- 0.4</td>\n",
       "      <td>56.0 +/- 0.2</td>\n",
       "      <td>9.5 +/- 0.2</td>\n",
       "      <td>73.8 +/- 0.1</td>\n",
       "      <td>52.8 +/- 0.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              clip          info         paint        quick  \\\n",
       "CLIPPretrained        60.4 +/- 0.1  38.5 +/- 0.3  54.7 +/- 0.2  8.2 +/- 0.1   \n",
       "CLIPBase              60.7 +/- 0.4  35.3 +/- 0.3  53.2 +/- 0.3  9.6 +/- 0.2   \n",
       "CLIPCondCAD_lambda_1  61.8 +/- 0.5  38.3 +/- 0.4  56.0 +/- 0.2  9.5 +/- 0.2   \n",
       "\n",
       "                              real        sketch  \n",
       "CLIPPretrained        72.8 +/- 0.1  51.1 +/- 0.2  \n",
       "CLIPBase              71.4 +/- 0.2  50.1 +/- 0.2  \n",
       "CLIPCondCAD_lambda_1  73.8 +/- 0.1  52.8 +/- 0.1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(resnet_result_dict['DomainNet']).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0ac4d0b3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clip</th>\n",
       "      <th>info</th>\n",
       "      <th>paint</th>\n",
       "      <th>quick</th>\n",
       "      <th>real</th>\n",
       "      <th>sketch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CLIPPretrained</th>\n",
       "      <td>70.6 +/- 0.2</td>\n",
       "      <td>38.1 +/- 0.3</td>\n",
       "      <td>60.6 +/- 0.3</td>\n",
       "      <td>12.8 +/- 0.2</td>\n",
       "      <td>76.5 +/- 0.2</td>\n",
       "      <td>58.2 +/- 0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPBase</th>\n",
       "      <td>70.7 +/- 0.1</td>\n",
       "      <td>36.9 +/- 0.3</td>\n",
       "      <td>59.8 +/- 0.2</td>\n",
       "      <td>14.3 +/- 0.1</td>\n",
       "      <td>75.4 +/- 0.1</td>\n",
       "      <td>57.4 +/- 0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPCondCAD_lambda_1e-1</th>\n",
       "      <td>71.3 +/- 0.2</td>\n",
       "      <td>38.3 +/- 0.6</td>\n",
       "      <td>61.5 +/- 0.2</td>\n",
       "      <td>14.3 +/- 0.1</td>\n",
       "      <td>77.5 +/- 0.1</td>\n",
       "      <td>59.3 +/- 0.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 clip          info         paint  \\\n",
       "CLIPPretrained           70.6 +/- 0.2  38.1 +/- 0.3  60.6 +/- 0.3   \n",
       "CLIPBase                 70.7 +/- 0.1  36.9 +/- 0.3  59.8 +/- 0.2   \n",
       "CLIPCondCAD_lambda_1e-1  71.3 +/- 0.2  38.3 +/- 0.6  61.5 +/- 0.2   \n",
       "\n",
       "                                quick          real        sketch  \n",
       "CLIPPretrained           12.8 +/- 0.2  76.5 +/- 0.2  58.2 +/- 0.1  \n",
       "CLIPBase                 14.3 +/- 0.1  75.4 +/- 0.1  57.4 +/- 0.3  \n",
       "CLIPCondCAD_lambda_1e-1  14.3 +/- 0.1  77.5 +/- 0.1  59.3 +/- 0.2  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(vit_result_dict['DomainNet']).T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d8e7440",
   "metadata": {},
   "source": [
    "## VLCS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7045a46b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>C</th>\n",
       "      <th>L</th>\n",
       "      <th>S</th>\n",
       "      <th>V</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CLIPPretrained</th>\n",
       "      <td>97.4 +/- 0.9</td>\n",
       "      <td>63.4 +/- 0.6</td>\n",
       "      <td>79.4 +/- 0.2</td>\n",
       "      <td>84.1 +/- 1.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPBase</th>\n",
       "      <td>98.1 +/- 0.9</td>\n",
       "      <td>64.6 +/- 1.4</td>\n",
       "      <td>80.3 +/- 0.2</td>\n",
       "      <td>83.4 +/- 0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPCondCAD_lambda_1e-2</th>\n",
       "      <td>98.3 +/- 0.3</td>\n",
       "      <td>66.1 +/- 1.2</td>\n",
       "      <td>80.6 +/- 0.3</td>\n",
       "      <td>84.3 +/- 0.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    C             L             S  \\\n",
       "CLIPPretrained           97.4 +/- 0.9  63.4 +/- 0.6  79.4 +/- 0.2   \n",
       "CLIPBase                 98.1 +/- 0.9  64.6 +/- 1.4  80.3 +/- 0.2   \n",
       "CLIPCondCAD_lambda_1e-2  98.3 +/- 0.3  66.1 +/- 1.2  80.6 +/- 0.3   \n",
       "\n",
       "                                    V  \n",
       "CLIPPretrained           84.1 +/- 1.6  \n",
       "CLIPBase                 83.4 +/- 0.9  \n",
       "CLIPCondCAD_lambda_1e-2  84.3 +/- 0.4  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(resnet_result_dict['VLCS']).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2ab13a3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>C</th>\n",
       "      <th>L</th>\n",
       "      <th>S</th>\n",
       "      <th>V</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CLIPPretrained</th>\n",
       "      <td>98.4 +/- 0.6</td>\n",
       "      <td>63.5 +/- 0.7</td>\n",
       "      <td>78.8 +/- 0.5</td>\n",
       "      <td>81.4 +/- 0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPBase</th>\n",
       "      <td>99.1 +/- 0.6</td>\n",
       "      <td>64.7 +/- 1.2</td>\n",
       "      <td>78.5 +/- 1.1</td>\n",
       "      <td>82.2 +/- 1.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPCondCAD_lambda_1e-2</th>\n",
       "      <td>99.0 +/- 0.3</td>\n",
       "      <td>65.3 +/- 1.3</td>\n",
       "      <td>78.8 +/- 0.5</td>\n",
       "      <td>83.4 +/- 1.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    C             L             S  \\\n",
       "CLIPPretrained           98.4 +/- 0.6  63.5 +/- 0.7  78.8 +/- 0.5   \n",
       "CLIPBase                 99.1 +/- 0.6  64.7 +/- 1.2  78.5 +/- 1.1   \n",
       "CLIPCondCAD_lambda_1e-2  99.0 +/- 0.3  65.3 +/- 1.3  78.8 +/- 0.5   \n",
       "\n",
       "                                    V  \n",
       "CLIPPretrained           81.4 +/- 0.9  \n",
       "CLIPBase                 82.2 +/- 1.9  \n",
       "CLIPCondCAD_lambda_1e-2  83.4 +/- 1.6  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(vit_result_dict['VLCS']).T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3624a38",
   "metadata": {},
   "source": [
    "## PACS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "346fa449",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>C</th>\n",
       "      <th>P</th>\n",
       "      <th>S</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CLIPPretrained</th>\n",
       "      <td>87.2 +/- 0.6</td>\n",
       "      <td>90.4 +/- 0.5</td>\n",
       "      <td>98.5 +/- 0.1</td>\n",
       "      <td>85.1 +/- 0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPBase</th>\n",
       "      <td>90.1 +/- 0.6</td>\n",
       "      <td>91.3 +/- 1.0</td>\n",
       "      <td>99.1 +/- 0.1</td>\n",
       "      <td>84.8 +/- 0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPCondCAD_lambda_1e-2</th>\n",
       "      <td>91.4 +/- 0.5</td>\n",
       "      <td>92.1 +/- 0.8</td>\n",
       "      <td>98.9 +/- 0.4</td>\n",
       "      <td>85.9 +/- 0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    A             C             P  \\\n",
       "CLIPPretrained           87.2 +/- 0.6  90.4 +/- 0.5  98.5 +/- 0.1   \n",
       "CLIPBase                 90.1 +/- 0.6  91.3 +/- 1.0  99.1 +/- 0.1   \n",
       "CLIPCondCAD_lambda_1e-2  91.4 +/- 0.5  92.1 +/- 0.8  98.9 +/- 0.4   \n",
       "\n",
       "                                    S  \n",
       "CLIPPretrained           85.1 +/- 0.4  \n",
       "CLIPBase                 84.8 +/- 0.4  \n",
       "CLIPCondCAD_lambda_1e-2  85.9 +/- 0.5  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(resnet_result_dict['PACS']).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "92080d30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>C</th>\n",
       "      <th>P</th>\n",
       "      <th>S</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CLIPPretrained</th>\n",
       "      <td>94.7 +/- 1.1</td>\n",
       "      <td>96.2 +/- 1.4</td>\n",
       "      <td>99.2 +/- 0.4</td>\n",
       "      <td>84.8 +/- 0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPBase</th>\n",
       "      <td>94.5 +/- 0.9</td>\n",
       "      <td>97.2 +/- 0.6</td>\n",
       "      <td>99.5 +/- 0.2</td>\n",
       "      <td>86.8 +/- 1.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPCondCAD_lambda_1e-2</th>\n",
       "      <td>95.8 +/- 0.4</td>\n",
       "      <td>97.7 +/- 0.3</td>\n",
       "      <td>99.4 +/- 0.2</td>\n",
       "      <td>86.7 +/- 0.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    A             C             P  \\\n",
       "CLIPPretrained           94.7 +/- 1.1  96.2 +/- 1.4  99.2 +/- 0.4   \n",
       "CLIPBase                 94.5 +/- 0.9  97.2 +/- 0.6  99.5 +/- 0.2   \n",
       "CLIPCondCAD_lambda_1e-2  95.8 +/- 0.4  97.7 +/- 0.3  99.4 +/- 0.2   \n",
       "\n",
       "                                    S  \n",
       "CLIPPretrained           84.8 +/- 0.9  \n",
       "CLIPBase                 86.8 +/- 1.1  \n",
       "CLIPCondCAD_lambda_1e-2  86.7 +/- 0.7  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(vit_result_dict['PACS']).T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "563a051b",
   "metadata": {},
   "source": [
    "## OfficeHome"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bcb1e4e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>C</th>\n",
       "      <th>P</th>\n",
       "      <th>R</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CLIPPretrained</th>\n",
       "      <td>68.1 +/- 0.4</td>\n",
       "      <td>50.1 +/- 0.7</td>\n",
       "      <td>82.3 +/- 0.2</td>\n",
       "      <td>82.0 +/- 0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPBase</th>\n",
       "      <td>68.4 +/- 1.6</td>\n",
       "      <td>50.6 +/- 0.4</td>\n",
       "      <td>81.7 +/- 0.3</td>\n",
       "      <td>81.8 +/- 0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPCondCAD_lambda_1e-2</th>\n",
       "      <td>69.6 +/- 1.6</td>\n",
       "      <td>51.2 +/- 1.1</td>\n",
       "      <td>82.2 +/- 0.3</td>\n",
       "      <td>82.2 +/- 0.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    A             C             P  \\\n",
       "CLIPPretrained           68.1 +/- 0.4  50.1 +/- 0.7  82.3 +/- 0.2   \n",
       "CLIPBase                 68.4 +/- 1.6  50.6 +/- 0.4  81.7 +/- 0.3   \n",
       "CLIPCondCAD_lambda_1e-2  69.6 +/- 1.6  51.2 +/- 1.1  82.2 +/- 0.3   \n",
       "\n",
       "                                    R  \n",
       "CLIPPretrained           82.0 +/- 0.3  \n",
       "CLIPBase                 81.8 +/- 0.9  \n",
       "CLIPCondCAD_lambda_1e-2  82.2 +/- 0.6  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(resnet_result_dict['OfficeHome']).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "04c8f35e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>C</th>\n",
       "      <th>P</th>\n",
       "      <th>R</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CLIPPretrained</th>\n",
       "      <td>76.8 +/- 0.5</td>\n",
       "      <td>67.2 +/- 0.5</td>\n",
       "      <td>87.6 +/- 0.3</td>\n",
       "      <td>87.8 +/- 0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPBase</th>\n",
       "      <td>76.2 +/- 0.9</td>\n",
       "      <td>67.3 +/- 0.4</td>\n",
       "      <td>87.3 +/- 0.3</td>\n",
       "      <td>87.4 +/- 0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPCondCAD_lambda_1e-2</th>\n",
       "      <td>76.2 +/- 0.7</td>\n",
       "      <td>68.2 +/- 0.7</td>\n",
       "      <td>88.1 +/- 0.2</td>\n",
       "      <td>87.6 +/- 0.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    A             C             P  \\\n",
       "CLIPPretrained           76.8 +/- 0.5  67.2 +/- 0.5  87.6 +/- 0.3   \n",
       "CLIPBase                 76.2 +/- 0.9  67.3 +/- 0.4  87.3 +/- 0.3   \n",
       "CLIPCondCAD_lambda_1e-2  76.2 +/- 0.7  68.2 +/- 0.7  88.1 +/- 0.2   \n",
       "\n",
       "                                    R  \n",
       "CLIPPretrained           87.8 +/- 0.3  \n",
       "CLIPBase                 87.4 +/- 0.4  \n",
       "CLIPCondCAD_lambda_1e-2  87.6 +/- 0.3  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(vit_result_dict['OfficeHome']).T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3cace334",
   "metadata": {},
   "source": [
    "## TerraIncognita"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "77bc9684",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>L100</th>\n",
       "      <th>L38</th>\n",
       "      <th>L43</th>\n",
       "      <th>L46</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CLIPPretrained</th>\n",
       "      <td>27.1 +/- 0.9</td>\n",
       "      <td>22.5 +/- 1.2</td>\n",
       "      <td>36.5 +/- 2.1</td>\n",
       "      <td>30.6 +/- 2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPBase</th>\n",
       "      <td>33.2 +/- 2.5</td>\n",
       "      <td>36.0 +/- 2.4</td>\n",
       "      <td>40.7 +/- 1.9</td>\n",
       "      <td>34.8 +/- 0.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPCondCAD_lambda_1e-2</th>\n",
       "      <td>35.0 +/- 2.5</td>\n",
       "      <td>34.3 +/- 3.7</td>\n",
       "      <td>40.4 +/- 1.4</td>\n",
       "      <td>34.5 +/- 1.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 L100           L38           L43  \\\n",
       "CLIPPretrained           27.1 +/- 0.9  22.5 +/- 1.2  36.5 +/- 2.1   \n",
       "CLIPBase                 33.2 +/- 2.5  36.0 +/- 2.4  40.7 +/- 1.9   \n",
       "CLIPCondCAD_lambda_1e-2  35.0 +/- 2.5  34.3 +/- 3.7  40.4 +/- 1.4   \n",
       "\n",
       "                                  L46  \n",
       "CLIPPretrained           30.6 +/- 2.5  \n",
       "CLIPBase                 34.8 +/- 0.7  \n",
       "CLIPCondCAD_lambda_1e-2  34.5 +/- 1.5  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(resnet_result_dict['TerraIncognita']).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8e48e8f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>L100</th>\n",
       "      <th>L38</th>\n",
       "      <th>L43</th>\n",
       "      <th>L46</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CLIPPretrained</th>\n",
       "      <td>42.5 +/- 1.5</td>\n",
       "      <td>36.7 +/- 0.4</td>\n",
       "      <td>38.9 +/- 1.0</td>\n",
       "      <td>29.3 +/- 0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPBase</th>\n",
       "      <td>46.6 +/- 0.7</td>\n",
       "      <td>46.4 +/- 2.2</td>\n",
       "      <td>39.5 +/- 0.4</td>\n",
       "      <td>29.5 +/- 1.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLIPCondCAD_lambda_1e-2</th>\n",
       "      <td>45.4 +/- 1.2</td>\n",
       "      <td>46.8 +/- 1.7</td>\n",
       "      <td>40.7 +/- 1.1</td>\n",
       "      <td>29.3 +/- 1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 L100           L38           L43  \\\n",
       "CLIPPretrained           42.5 +/- 1.5  36.7 +/- 0.4  38.9 +/- 1.0   \n",
       "CLIPBase                 46.6 +/- 0.7  46.4 +/- 2.2  39.5 +/- 0.4   \n",
       "CLIPCondCAD_lambda_1e-2  45.4 +/- 1.2  46.8 +/- 1.7  40.7 +/- 1.1   \n",
       "\n",
       "                                  L46  \n",
       "CLIPPretrained           29.3 +/- 0.8  \n",
       "CLIPBase                 29.5 +/- 1.8  \n",
       "CLIPCondCAD_lambda_1e-2  29.3 +/- 1.0  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(vit_result_dict['TerraIncognita']).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "neither-grade",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "optdom",
   "language": "python",
   "name": "optdom"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}