Merge branch 'open-mmlab:main' into zyc/videochat

open-mmlab · Sep 20, 2023 · 8cfcd00 · 8cfcd00
2 parents 63e3c37 + d36db71
commit 8cfcd00
Show file tree

Hide file tree

Showing 20 changed files with 63 additions and 56 deletions.
diff --git a/configs/_base_/models/tin_r50.py b/configs/_base_/models/tin_r50.py
@@ -1,7 +1,9 @@
 # model settings
 
 preprocess_cfg = dict(
-    mean=[127.5, 127.5, 127.5], std=[127.5, 127.5, 127.5], format_shape='NCHW')
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    format_shape='NCHW')
 
 model = dict(
     type='Recognizer2D',

diff --git a/configs/recognition/tin/README.md b/configs/recognition/tin/README.md
@@ -34,7 +34,7 @@ For a long time, the vision community tries to learn the spatio-temporal represe
 
 | frame sampling strategy |   resolution   | gpus | backbone |    pretrain     | top1 acc | top5 acc | testing protocol | inference time(video/s) | gpu_mem(M) |          config           |          ckpt           |           log           |
 | :---------------------: | :------------: | :--: | :------: | :-------------: | :------: | :------: | :--------------: | :---------------------: | :--------: | :-----------------------: | :---------------------: | :---------------------: |
-|          1x1x8          | short-side 256 | 8x4  | ResNet50 | TSM-Kinetics400 |  71.77   |  90.36   | 8 clips x 1 crop |            x            |    6185    | [config](/configs/recognition/tin/tin_imagenet-pretrained-r50_8xb6-1x1x8-40e_sthv2-rgb.py) | [ckpt](https://download.openmmlab.com/mmaction/v1.0/recognition/tin/tin_kinetics400-pretrained-tsm-r50_1x1x8-50e_kinetics400-rgb/tin_kinetics400-pretrained-tsm-r50_1x1x8-50e_kinetics400-rgb_20220913-7f10d0c0.pth) | [log](https://download.openmmlab.com/mmaction/v1.0/recognition/tin/tin_kinetics400-pretrained-tsm-r50_1x1x8-50e_kinetics400-rgb/tin_kinetics400-pretrained-tsm-r50_1x1x8-50e_kinetics400-rgb.log) |
+|          1x1x8          | short-side 256 | 8x4  | ResNet50 | TSM-Kinetics400 |  71.86   |  90.44   | 8 clips x 1 crop |            x            |    6185    | [config](/configs/recognition/tin/tin_imagenet-pretrained-r50_8xb6-1x1x8-40e_sthv2-rgb.py) | [ckpt](https://download.openmmlab.com/mmaction/v1.0/recognition/tin/tin_kinetics400-pretrained-tsm-r50_1x1x8-50e_kinetics400-rgb/tin_kinetics400-pretrained-tsm-r50_1x1x8-50e_kinetics400-rgb_20220913-7f10d0c0.pth) | [log](https://download.openmmlab.com/mmaction/v1.0/recognition/tin/tin_kinetics400-pretrained-tsm-r50_1x1x8-50e_kinetics400-rgb/tin_kinetics400-pretrained-tsm-r50_1x1x8-50e_kinetics400-rgb.log) |
 
 Here, we use `finetune` to indicate that we use [TSM model](https://download.openmmlab.com/mmaction/v1.0/v1.0/recognition/tsm/tsm_imagenet-pretrained-r50_8xb16-1x1x8-50e_kinetics400-rgb/tsm_imagenet-pretrained-r50_8xb16-1x1x8-50e_kinetics400-rgb_20220831-64d69186.pth) trained on Kinetics-400 to finetune the TIN model on Kinetics-400.
 

diff --git a/configs/recognition/tin/metafile.yml b/configs/recognition/tin/metafile.yml
@@ -66,8 +66,8 @@ Models:
   Results:
   - Dataset: Kinetics-400
     Metrics:
-      Top 1 Accuracy: 71.77
-      Top 5 Accuracy: 90.36
+      Top 1 Accuracy: 71.86
+      Top 5 Accuracy: 90.44
     Task: Action Recognition
   Training Log: https://download.openmmlab.com/mmaction/v1.0/recognition/tin/tin_kinetics400-pretrained-tsm-r50_1x1x8-50e_kinetics400-rgb/tin_kinetics400-pretrained-tsm-r50_1x1x8-50e_kinetics400-rgb.log
   Weights: https://download.openmmlab.com/mmaction/v1.0/recognition/tin/tin_kinetics400-pretrained-tsm-r50_1x1x8-50e_kinetics400-rgb/tin_kinetics400-pretrained-tsm-r50_1x1x8-50e_kinetics400-rgb_20220913-7f10d0c0.pth
diff --git a/dataset-index.yml b/dataset-index.yml
@@ -1,39 +1,40 @@
+openxlab: true
 kinetics400:
-  dataset: Kinetics-400
+  dataset: OpenMMLab/Kinetics-400
   download_root: data
   data_root: data/kinetics400
-  script: tools/data/kinetics/k400_preprocess.sh
+  script: tools/data/kinetics/preprocess_k400.sh
 
 kinetics600:
-  dataset: Kinetics600
+  dataset: OpenMMLab/Kinetics600
   download_root: data
   data_root: data/kinetics600
-  script: tools/data/kinetics/k600_preprocess.sh
+  script: tools/data/kinetics/preprocess_k600.sh
 
 kinetics700:
-  dataset: Kinetics_700
+  dataset: OpenMMLab/Kinetics_700
   download_root: data
   data_root: data/kinetics700
-  script: tools/data/kinetics/k700_preprocess.sh
+  script: tools/data/kinetics/preprocess_k700.sh
 
 sthv2:
-  dataset: sthv2
+  dataset: OpenDataLab/sthv2
   download_root: data
   data_root: data/sthv2
   script: tools/data/sthv2/preprocess.sh
 
 ucf-101:
-  dataset: UCF101
+  dataset: OpenDataLab/UCF101
   download_root: data
   data_root: data/ucf101
 
 finegym:
-  dataset: FineGym
+  dataset: OpenDataLab/FineGym
   download_root: data
   data_root: data/gym
 
 diving48:
-  dataset: diving48
+  dataset: OpenDataLab/diving48
   download_root: data
   data_root: data/diving48
   script: tools/data/diving48/preprocess.sh
diff --git a/mmaction/models/backbones/resnet_tin.py b/mmaction/models/backbones/resnet_tin.py
@@ -325,6 +325,9 @@ def init_structure(self):
         if len(self.non_local_cfg) != 0:
             self.make_non_local()
 
+    def _get_wrap_prefix(self):
+        return ['.net2']
+
     def make_temporal_interlace(self):
         """Make temporal interlace for some layers."""
         num_segment_list = [self.num_segments] * 4
@@ -365,6 +368,3 @@ def make_block_interlace(stage, num_segments, shift_div):
                                            self.shift_div)
         self.layer4 = make_block_interlace(self.layer4, num_segment_list[3],
                                            self.shift_div)
-
-    def init_weights(self):
-        pass
diff --git a/mmaction/models/backbones/resnet_tsm.py b/mmaction/models/backbones/resnet_tsm.py
@@ -305,6 +305,9 @@ def make_non_local(self):
                                                  self.num_segments,
                                                  self.non_local_cfg)
 
+    def _get_wrap_prefix(self):
+        return ['.net', '.block']
+
     def load_original_weights(self, logger):
         """Load weights from original checkpoint, which required converting
         keys."""
@@ -317,7 +320,7 @@ def load_original_weights(self, logger):
         for name, module in self.named_modules():
             # convert torchvision keys
             ori_name = name
-            for wrap_prefix in ['.net', '.block']:
+            for wrap_prefix in self._get_wrap_prefix():
                 if wrap_prefix in ori_name:
                     ori_name = ori_name.replace(wrap_prefix, '')
                     wrapped_layers_map[ori_name] = name
@@ -352,6 +355,7 @@ def load_original_weights(self, logger):
             if layer_name in wrapped_layers_map:
                 wrapped_name = param_name.replace(
                     layer_name, wrapped_layers_map[layer_name])
+                print(f'wrapped_name {wrapped_name}')
                 state_dict_torchvision[
                     wrapped_name] = state_dict_torchvision.pop(param_name)
 

diff --git a/projects/gesture_recognition/README.md b/projects/gesture_recognition/README.md
@@ -14,7 +14,7 @@ Hand detection results on OneHand10K validation dataset
 
 | Config                                                  | Input Size | bbox mAP | bbox mAP 50 | bbox mAP 75 |                         ckpt                          |                         log                          |
 | :------------------------------------------------------ | :--------: | :------: | :---------: | :---------: | :---------------------------------------------------: | :--------------------------------------------------: |
-| [rtmpose_nano](/projects/gesture_recognition/configs/rtmdet-nano_8xb32-300e_multi-dataset-hand-320x320.py) |  320x320   |  0.8100  |   0.9870    |   0.9190    | [ckpt](https://download.openmmlab.com/mmaction/v1.0/projects/gesture_recognition/rtmdet-nano_8xb32-300e_multi-dataset-hand-320x320_20230524-f6ffed6a.pth) | [log](https://download.openmmlab.com/mmaction/v1.0/projects/gesture_recognition/rtmdet-nano_8xb32-300e_multi-dataset-hand-320x320.log) |
+| [rtmdet_nano](/projects/gesture_recognition/configs/rtmdet-nano_8xb32-300e_multi-dataset-hand-320x320.py) |  320x320   |  0.8100  |   0.9870    |   0.9190    | [ckpt](https://download.openmmlab.com/mmaction/v1.0/projects/gesture_recognition/rtmdet-nano_8xb32-300e_multi-dataset-hand-320x320_20230524-f6ffed6a.pth) | [log](https://download.openmmlab.com/mmaction/v1.0/projects/gesture_recognition/rtmdet-nano_8xb32-300e_multi-dataset-hand-320x320.log) |
 
 ## Pose estimation stage
 

diff --git a/tools/data/diving48/README.md b/tools/data/diving48/README.md
@@ -21,10 +21,10 @@ For basic dataset information, you can refer to the official dataset [website](h
 ````{group-tab} Download by MIM
 MIM supports downloading from OpenDataLab and preprocessing Diving48 dataset with one command line.
 ```Bash
-# install OpenDataLab CLI tools
-pip install -U opendatalab
-# log in OpenDataLab
-odl login
+# install OpenXlab CLI tools
+pip install -U openxlab
+# log in OpenXLab
+openxlab login
 # download and preprocess by MIM
 mim download mmaction2 --dataset diving48
 ```

diff --git a/tools/data/diving48/README_zh-CN.md b/tools/data/diving48/README_zh-CN.md
@@ -21,10 +21,10 @@
 ````{group-tab} 使用 MIM 下载
 # MIM 支持下载 Diving48 数据集。用户可以通过一行命令，从 OpenDataLab 进行下载，并进行预处理。
 ```Bash
-# 安装 OpenDataLab CLI 工具
-pip install -U opendatalab
-# 登录 OpenDataLab
-odl login
+# 安装 OpenXLab CLI 工具
+pip install -U openxlab
+# 登录 OpenXLab
+openxlab login
 # 通过 MIM 进行数据集下载，预处理。注意这将花费较长时间
 mim download mmaction2 --dataset diving48
 ```

diff --git a/tools/data/diving48/preprocess.sh b/tools/data/diving48/preprocess.sh
@@ -3,6 +3,6 @@
 DOWNLOAD_DIR=$1
 DATA_ROOT=$2
 
-cat $DOWNLOAD_DIR/diving48/raw/*.tar.gz.*  | tar -xvz -C $(dirname $DATA_ROOT)
+cat $DOWNLOAD_DIR/OpenDataLab___diving48/raw/*.tar.gz.*  | tar -xvz -C $(dirname $DATA_ROOT)
 tar -xvf $DATA_ROOT/diving48.tar -C $(dirname $DATA_ROOT)
 rm $DATA_ROOT/diving48.tar
diff --git a/tools/data/kinetics/README.md b/tools/data/kinetics/README.md
@@ -38,10 +38,10 @@ All experiments on Kinetics in MMAction2 are based on this version, we recommend
 MIM supports downloading from OpenDataLab and preprocessing Kinetics-400/600/700 dataset with one command line.
 
 ```Bash
-# install OpenDataLab CLI tools
-pip install -U opendatalab
-# log in OpenDataLab
-odl login
+# install OpenXlab CLI tools
+pip install -U openxlab
+# log in OpenXLab
+openxlab login
 # download and preprocess Kinetics-400 by MIM. Note that this might take a long time.
 mim download mmaction2 --dataset kinetics400
 # download and preprocess Kinetics-600 by MIM. Note that this might take a long time.

diff --git a/tools/data/kinetics/README_zh-CN.md b/tools/data/kinetics/README_zh-CN.md
@@ -36,10 +36,10 @@ MMAction2 代码仓库中提供的 Kinetics 实验性能，都是基于这个版
 
 # MIM 支持下载 Kinetics-400/600/700 数据集。用户可以通过一行命令，从 OpenDataLab 进行下载，并进行预处理。
 ```Bash
-# 安装 OpenDataLab CLI 工具
-pip install -U opendatalab
-# 登录 OpenDataLab
-odl login
+# 安装 OpenXLab CLI 工具
+pip install -U openxlab
+# 登录 OpenXLab
+openxlab login
 # 通过 MIM 进行 Kinetics-400 数据集下载，预处理。注意这将花费较长时间
 mim download mmaction2 --dataset kinetics400
 # 通过 MIM 进行 Kinetics-600 数据集下载，预处理。注意这将花费较长时间

diff --git a/tools/data/kinetics/preprocess_k400.sh b/tools/data/kinetics/preprocess_k400.sh
@@ -5,5 +5,5 @@ set -x
 DOWNLOAD_DIR=$1
 DATA_ROOT=$2
 
-cat $DOWNLOAD_DIR/Kinetics-400/raw/*.tar.gz*  | tar -xvz -C $(dirname $DATA_ROOT)
+cat $DOWNLOAD_DIR/OpenMMLab___Kinetics-400/raw/*.tar.gz*  | tar -xvz -C $(dirname $DATA_ROOT)
 mv $(dirname $DATA_ROOT)/Kinetics-400 $DATA_ROOT
diff --git a/tools/data/kinetics/preprocess_k600.sh b/tools/data/kinetics/preprocess_k600.sh
@@ -5,5 +5,5 @@ set -x
 DOWNLOAD_DIR=$1
 DATA_ROOT=$2
 
-cat $DOWNLOAD_DIR/Kinetics600/raw/*.tar.gz*  | tar -xvz -C $(dirname $DATA_ROOT)
+cat $DOWNLOAD_DIR/OpenMMLab___Kinetics600/raw/*.tar.gz*  | tar -xvz -C $(dirname $DATA_ROOT)
 mv $(dirname $DATA_ROOT)/Kinetics600 $DATA_ROOT
diff --git a/tools/data/kinetics/preprocess_k700.sh b/tools/data/kinetics/preprocess_k700.sh
@@ -5,5 +5,5 @@ set -x
 DOWNLOAD_DIR=$1
 DATA_ROOT=$2
 
-cat $DOWNLOAD_DIR/Kinetics_700/raw/*.tar.gz*  | tar -xvz -C $(dirname $DATA_ROOT)
+cat $DOWNLOAD_DIR/OpenMMLab___Kinetics_700/raw/*.tar.gz*  | tar -xvz -C $(dirname $DATA_ROOT)
 mv $(dirname $DATA_ROOT)/Kinetics_700 $DATA_ROOT
diff --git a/tools/data/kinetics710/README.md b/tools/data/kinetics710/README.md
@@ -25,10 +25,10 @@ Before we start, please make sure that the directory is located at `$MMACTION2`.
 Kinetics-710 is a video benchmark based on Kinetics-400/600/700, which merges the training set of these Kinetics datasets, and deletes the repeated videos according to Youtube IDs. MMAction2 provides an annotation file based on the Kinetics-400/600/700 on [OpenDataLab](https://opendatalab.com/). So we suggest you download Kinetics-400/600/700 first from OpenDataLab by [MIM](https://github.com/open-mmlab/mim).
 
 ```shell
-# install OpenDataLab CLI tools
-pip install -U opendatalab
-# log in OpenDataLab
-odl login
+# install OpenXlab CLI tools
+pip install -U openxlab
+# log in OpenXLab
+openxlab login
 # download Kinetics-400/600/700, note that this might take a long time.
 mim download mmaction2 --dataset kinetics400
 mim download mmaction2 --dataset kinetics600

diff --git a/tools/data/kinetics710/README_zh-CN.md b/tools/data/kinetics710/README_zh-CN.md
@@ -23,10 +23,10 @@
 Kinetics-710 是基于 Kinetics-400/600/700 的视频数据集，它合并了这些 Kinetics 数据集的训练集，并根据 Youtube ID 删除了重复的视频。MMAction2 提供了一个基于 Kinetics-400/600/700 的 OpenDataLab 版本的标注文件，你可以通过 [MIM](https://github.com/open-mmlab/mim) 从 OpenDataLab 下载。
 
 ```shell
-# 安装 OpenDataLab CLI 工具
-pip install -U opendatalab
-# 登录 OpenDataLab
-odl login
+# 安装 OpenXLab CLI 工具
+pip install -U openxlab
+# 登录 OpenXLab
+openxlab login
 # 下载 Kinetics-400/600/700，注意这可能需要很长时间。
 mim download mmaction2 --dataset kinetics400
 mim download mmaction2 --dataset kinetics600

diff --git a/tools/data/sthv2/README.md b/tools/data/sthv2/README.md
@@ -22,10 +22,10 @@ For basic dataset information, you can refer to the dataset [website](https://de
 ````{group-tab} Download by MIM
 MIM supports downloading from OpenDataLab and preprocessing Something-Something V2 dataset with one command line.
 ```Bash
-# install OpenDataLab CLI tools
-pip install -U opendatalab
-# log in OpenDataLab
-odl login
+# install OpenXlab CLI tools
+pip install -U openxlab
+# log in OpenXLab
+openxlab login
 # download and preprocess by MIM
 mim download mmaction2 --dataset sthv2
 ```

diff --git a/tools/data/sthv2/README_zh-CN.md b/tools/data/sthv2/README_zh-CN.md
@@ -22,10 +22,10 @@
 ````{group-tab} 使用 MIM 下载
 # MIM 支持下载 Something-Something V2 数据集。用户可以通过一行命令，从 OpenDataLab 进行下载，并进行预处理。
 ```Bash
-# 安装 OpenDataLab CLI 工具
-pip install -U opendatalab
-# 登录 OpenDataLab
-odl login
+# 安装 OpenXLab CLI 工具
+pip install -U openxlab
+# 登录 OpenXLab
+openxlab login
 # 通过 MIM 进行数据集下载，预处理。注意这将花费较长时间
 mim download mmaction2 --dataset sthv2
 ```

diff --git a/tools/data/sthv2/preprocss.sh b/tools/data/sthv2/preprocss.sh
@@ -3,6 +3,6 @@
 DOWNLOAD_DIR=$1
 DATA_ROOT=$2
 
-cat $DOWNLOAD_DIR/sthv2/raw/*.tar.gz  | tar -xvz -C $(dirname $DATA_ROOT)
+cat $DOWNLOAD_DIR/OpenDataLab___sthv2/raw/*.tar.gz  | tar -xvz -C $(dirname $DATA_ROOT)
 tar -xvf $DATA_ROOT/sthv2.tar -C $(dirname $DATA_ROOT)
 rm $DATA_ROOT/sthv2.tar