[Feature] Add PoseFormer backbone

configs/body/3d_kpt_sview_rgb_vid/video_pose_lift/h36m/poseformer_h36m_81frame_cpn.py

@@ -0,0 +1,150 @@
+_base_ = [
+    '../../../../_base_/default_runtime.py',
+    '../../../../_base_/datasets/h36m.py'
+]
+evaluation = dict(
+    interval=10, metric=['mpjpe', 'p-mpjpe'], key_indicator='MPJPE')
+
+# optimizer settings
+optimizer = dict(
+    type='Adam',
+    lr=2e-4,
+    weight_decay=0.1
+)
+optimizer_config = dict(grad_clip=None)
+# learning policy
+lr_config = dict(
+    policy='exp',
+    by_epoch=True,
+    gamma=0.98,
+)
+
+total_epochs = 130
+
+log_config = dict(
+    interval=20,
+    hooks=[
+        dict(type='TextLoggerHook'),
+        # dict(type='TensorboardLoggerHook')
+    ])
+
+channel_cfg = dict(
+    num_output_channels=17,
+    dataset_joints=17,
+    dataset_channel=[
+        [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
+    ],
+    inference_channel=[
+        0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16
+    ])
+
+# model settings
+model = dict(
+    type='PoseLifter',
+    pretrained=None,
+    backbone=dict(
+        type='PoseFormer', num_frame=81, drop_path_rate=0.1),
+    keypoint_head=dict(
+        type='PoseFormerHead', loss_keypoint=dict(type='MPJPELoss')),
+    train_cfg=dict(),
+    test_cfg=dict(restore_global_position=True))
+
+# data settings
+data_root = 'data/h36m'
+train_data_cfg = dict(
+    num_joints=17,
+    seq_len=81,
+    seq_frame_interval=1,
+    causal=False,
+    temporal_padding=False,
+    joint_2d_src='detection',
+    joint_2d_det_file=f'{data_root}/joint_2d_det_files/' +
+    'cpn_ft_h36m_dbb_train.npy',
+    need_camera_param=True,
+    camera_param_file=f'{data_root}/annotation_body3d/cameras.pkl',
+)
+test_data_cfg = dict(
+    num_joints=17,
+    seq_len=81,
+    seq_frame_interval=1,
+    causal=False,
+    temporal_padding=False,
+    joint_2d_src='detection',
+    joint_2d_det_file=f'{data_root}/joint_2d_det_files/' +
+    'cpn_ft_h36m_dbb_test.npy',
+    need_camera_param=True,
+    camera_param_file=f'{data_root}/annotation_body3d/cameras.pkl',
+)
+
+train_pipeline = [
+    dict(
+        type='GetRootCenteredPose',
+        item='target',
+        visible_item='target_visible',
+        root_index=0,
+        root_name='root_position',
+        remove_root=False),
+    dict(type='ImageCoordinateNormalization', item='input_2d'),
+    dict(
+        type='RelativeJointRandomFlip',
+        item=['input_2d', 'target'],
+        flip_cfg=[
+            dict(center_mode='static', center_x=0.),
+            dict(center_mode='root', center_index=0)
+        ],
+        visible_item=['input_2d_visible', 'target_visible'],
+        flip_prob=0.5),
+    dict(type='PoseSequenceToTensor', item='input_2d', reshape=False),
+    dict(
+        type='Collect',
+        keys=[('input_2d', 'input'), 'target'],
+        meta_name='metas',
+        meta_keys=['target_image_path', 'flip_pairs', 'root_position'])
+]
+
+val_pipeline = [
+    dict(
+        type='GetRootCenteredPose',
+        item='target',
+        visible_item='target_visible',
+        root_index=0,
+        root_name='root_position',
+        remove_root=False),
+    dict(type='ImageCoordinateNormalization', item='input_2d'),
+    dict(type='PoseSequenceToTensor', item='input_2d', reshape=False),
+    dict(
+        type='Collect',
+        keys=[('input_2d', 'input'), 'target'],
+        meta_name='metas',
+        meta_keys=['target_image_path', 'flip_pairs', 'root_position'])
+]
+
+test_pipeline = val_pipeline
+
+data = dict(
+    samples_per_gpu=128,
+    workers_per_gpu=2,
+    val_dataloader=dict(samples_per_gpu=128),
+    test_dataloader=dict(samples_per_gpu=128),
+    train=dict(
+        type='Body3DH36MDataset',
+        ann_file=f'{data_root}/annotation_body3d/fps50/h36m_train.npz',
+        img_prefix=f'{data_root}/images/',
+        data_cfg=train_data_cfg,
+        pipeline=train_pipeline,
+        dataset_info={{_base_.dataset_info}}),
+    val=dict(
+        type='Body3DH36MDataset',
+        ann_file=f'{data_root}/annotation_body3d/fps50/h36m_test.npz',
+        img_prefix=f'{data_root}/images/',
+        data_cfg=test_data_cfg,
+        pipeline=val_pipeline,
+        dataset_info={{_base_.dataset_info}}),
+    test=dict(
+        type='Body3DH36MDataset',
+        ann_file=f'{data_root}/annotation_body3d/fps50/h36m_test.npz',
+        img_prefix=f'{data_root}/images/',
+        data_cfg=test_data_cfg,
+        pipeline=test_pipeline,
+        dataset_info={{_base_.dataset_info}}),
+)

mmpose/datasets/pipelines/pose3d_transform.py

@@ -446,8 +446,9 @@ class PoseSequenceToTensor:
         item
     """
 
-    def __init__(self, item):
+    def __init__(self, item, reshape=True):
         self.item = item
+        self.reshape = reshape
 
     def __call__(self, results):
         assert self.item in results
@@ -459,8 +460,9 @@ def __call__(self, results):
         if seq.ndim == 2:
             seq = seq[None, ...]
 
-        T = seq.shape[0]
-        seq = seq.transpose(1, 2, 0).reshape(-1, T)
+        if self.reshape:
+            T = seq.shape[0]
+            seq = seq.transpose(1, 2, 0).reshape(-1, T)
         results[self.item] = torch.from_numpy(seq)
 
         return results

mmpose/models/backbones/__init__.py

@@ -9,6 +9,7 @@
 from .mobilenet_v2 import MobileNetV2
 from .mobilenet_v3 import MobileNetV3
 from .mspn import MSPN
+from .poseformer import PoseFormer
 from .regnet import RegNet
 from .resnest import ResNeSt
 from .resnet import ResNet, ResNetV1d
@@ -30,5 +31,5 @@
     'MobileNetV3', 'RegNet', 'ResNet', 'ResNetV1d', 'ResNeXt', 'SCNet',
     'SEResNet', 'SEResNeXt', 'ShuffleNetV1', 'ShuffleNetV2', 'CPM', 'RSN',
     'MSPN', 'ResNeSt', 'VGG', 'TCN', 'ViPNAS_ResNet', 'ViPNAS_MobileNetV3',
-    'LiteHRNet', 'V2VNet', 'HRFormer'
+    'LiteHRNet', 'V2VNet', 'HRFormer', 'PoseFormer'
 ]