前言

最近两天在研究MANO模型的基本使用，从网上下载了MANO模型的模型文件和实例代码和Freihand数据集。数据集中的training_mano.json即为训练集数据的mano参数，共32560条数据，每条数据都是一个61维的向量。其中，pose：[:48]，shape：[48:58]。

目的是使用真实的数据搭配MANO进行3D的重建。

问题

在MANO附带的代码中，使用了PCA降维的操作，使得输入的形状参数减少到6个，不包括全局旋转，代码如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
from webuser.smpl_handpca_wrapper_HAND_only import load_model
import numpy as np

## Load MANO/SMPL+H model (here we load the righ hand model)
## Make sure path is correct
m = load_model('../../models/MANO_RIGHT.pkl', ncomps=6, flat_hand_mean=False)

## Assign random pose and shape parameters
m.betas[:] = np.random.rand(m.betas.size) * .03
#m.pose[:] = np.random.rand(m.pose.size) * 1.0
m.pose[:3] = [0., 0., 0.]
m.pose[3:] = [-0.42671473, -0.85829819, -0.50662164, +1.97374622, -0.84298473, -1.29958491]
# the first 3 elements correspond to global rotation
# the next ncomps to the hand pose

## Write to an .obj file
outmesh_path = './MANO___hello_world___PosedShaped.obj'
with open(outmesh_path, 'w') as fp:
    for v in m.r:
        fp.write( 'v %f %f %f\n' % ( v[0], v[1], v[2]) )

    for f in m.f+1: # Faces are 1-based, not 0-based in obj files
        fp.write( 'f %d %d %d\n' %  (f[0], f[1], f[2]) )

## Print message
print '..Output mesh saved to: ', outmesh_path

为此我将ncomps修改为了45（不包括全局旋转），同时传入了数据集中的真实值：

1
2
3
4
5
6
7
8
9
ncomps = 45
mano = np.array(json.load(open('./training_mano.json','r')))
# 要重建的图片索引
index = 236
mano_params = mano[index][0]

# 自己改的代码
m.betas[:] = mano_params[48:58] # shape
m.pose[:] = mano_params[:48]    # pose

然而，输入的结果却并不符合实际！

使用manopth是可以正常渲染出来的，这说明参数传入的没有问题，问题还是在代码上。

解决方法

苦寻无果，我查看了freihand数据集的仓库https://github.com/lmb-freiburg/freihand，

注意到仓库中的setup_mano.py，该脚本对MANO中的代码进行了修改，首先进行复制重命名操作：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# files we attempt to copy from the original mano repository
    files_needed = [
        'models/MANO_RIGHT.pkl',
        'webuser/verts.py',
        'webuser/posemapper.py',
        'webuser/lbs.py',
        'webuser/smpl_handpca_wrapper_HAND_only.py',
    ]

    # how to rename them to be used in our repository
    files_copy_to = [
        'data/MANO_RIGHT.pkl',
        'utils/mano_core/verts.py',
        'utils/mano_core/posemapper.py',
        'utils/mano_core/lbs.py',
        'utils/mano_core/mano_loader.py',
    ]

可以看到smpl_handpca_wrapper_HAND_only.py被重命名成了mano_loader.py。

然后修改了mano_loader.py中的代码：

1
2
3
4
5
6
7
8
9
10
11
12
def _patch_mano_loader():
    file = 'utils/mano_core/mano_loader.py'

    replace(file, *zip(*
        [
            (26, '    from posemapper import posemap'),
            (62, 'def load_model(fname_or_dict, ncomps=6, flat_hand_mean=False, v_template=None, use_pca=True):'),
            (66, '    from verts import verts_core'),
            (80, '    if use_pca:\n        hands_components = smpl_data[''hands_components'']  # PCA components\n    else:\n        hands_components = np.eye(45)  # directly modify 15x3 articulation angles'),
            (131,'    result.dd = dd'),
        ]
    ))

修改后的代码如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
def ready_arguments(fname_or_dict, posekey4vposed='pose'):
    import numpy as np
    import pickle
    import chumpy as ch
    from chumpy.ch import MatVecMult
    from mano.webuser.posemapper import posemap

    if not isinstance(fname_or_dict, dict):
        dd = pickle.load(open(fname_or_dict))
    else:
        dd = fname_or_dict

    want_shapemodel = 'shapedirs' in dd
    nposeparms = dd['kintree_table'].shape[1]*3

    if 'trans' not in dd:
        dd['trans'] = np.zeros(3)
    if 'pose' not in dd:
        dd['pose'] = np.zeros(nposeparms)
    if 'shapedirs' in dd and 'betas' not in dd:
        dd['betas'] = np.zeros(dd['shapedirs'].shape[-1])

    for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']:
        if (s in dd) and not hasattr(dd[s], 'dterms'):
            dd[s] = ch.array(dd[s])

    assert(posekey4vposed in dd)
    if want_shapemodel:
        dd['v_shaped'] = dd['shapedirs'].dot(dd['betas'])+dd['v_template']
        v_shaped = dd['v_shaped']
        J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:, 0])
        J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:, 1])
        J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:, 2])
        dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
        dd['v_posed'] = v_shaped + dd['posedirs'].dot(posemap(dd['bs_type'])(dd[posekey4vposed]))
    else:
        dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(posemap(dd['bs_type'])(dd[posekey4vposed]))

    return dd


def load_model(fname_or_dict, ncomps=6, flat_hand_mean=False, v_template=None, use_pca=True):
    ''' This model loads the fully articulable HAND SMPL model,
    and replaces the pose DOFS by ncomps from PCA'''

    from mano.webuser.verts import verts_core
    import numpy as np
    import chumpy as ch
    import pickle
    import scipy.sparse as sp
    np.random.seed(1)

    if not isinstance(fname_or_dict, dict):
        smpl_data = pickle.load(open(fname_or_dict,'rb'),encoding='latin1')
    else:
        smpl_data = fname_or_dict

    rot = 3  # for global orientation!!!

    if use_pca:
        hands_components = smpl_data['hands_components']  # PCA components
    else:
        hands_components = np.eye(45)  # directly modify 15x3 articulation angles
    hands_mean       = np.zeros(hands_components.shape[1]) if flat_hand_mean else smpl_data['hands_mean']
    hands_coeffs     = smpl_data['hands_coeffs'][:, :ncomps]

    selected_components = np.vstack((hands_components[:ncomps]))
    hands_mean = hands_mean.copy()

    pose_coeffs = ch.zeros(rot + selected_components.shape[0])
    full_hand_pose = pose_coeffs[rot:(rot+ncomps)].dot(selected_components)

    smpl_data['fullpose'] = ch.concatenate((pose_coeffs[:rot], hands_mean + full_hand_pose))
    smpl_data['pose'] = pose_coeffs

    Jreg = smpl_data['J_regressor']
    if not sp.issparse(Jreg):
        smpl_data['J_regressor'] = (sp.csc_matrix((Jreg.data, (Jreg.row, Jreg.col)), shape=Jreg.shape))

    # slightly modify ready_arguments to make sure that it uses the fullpose 
    # (which will NOT be pose) for the computation of posedirs
    dd = ready_arguments(smpl_data, posekey4vposed='fullpose')

    # create the smpl formula with the fullpose,
    # but expose the PCA coefficients as smpl.pose for compatibility
    args = {
        'pose': dd['fullpose'],
        'v': dd['v_posed'],
        'J': dd['J'],
        'weights': dd['weights'],
        'kintree_table': dd['kintree_table'],
        'xp': ch,
        'want_Jtr': True,
        'bs_style': dd['bs_style'],
    }

    result_previous, meta = verts_core(**args)
    result = result_previous + dd['trans'].reshape((1, 3))
    result.no_translation = result_previous

    if meta is not None:
        for field in ['Jtr', 'A', 'A_global', 'A_weighted']:
            if(hasattr(meta, field)):
                setattr(result, field, getattr(meta, field))

    if hasattr(result, 'Jtr'):
        result.J_transformed = result.Jtr + dd['trans'].reshape((1, 3))

    for k, v in dd.items():
        setattr(result, k, v)

    if v_template is not None:
        result.v_template[:] = v_template
    result.dd = dd
    return result

if __name__ == '__main__':
    m = load_model()
    m.J_transformed
    # print 'FINITO'

将修改后的代码替换掉原来的smpl_handpca_wrapper_HAND_only.py，在执行一开始的代码，就可以重建出正确的3D手部姿态了。

其他问题

如果遇到gbk无法读取pkl的问题，修改代码如下：

1
smpl_data = pickle.load(open(fname_or_dict,'rb'),encoding='latin1')

在open函数中加入rb模式，同时设置load的编码方式latin1

在windows平台python3环境下：

1
import cPickle as pickle

改为

1
import pickle