VR Prediction and Visualization

[wwwpkol]

Hello,

I am currently working on a project involving VR prediction and visualization using HTC Vive Tracking to capture positions and rotations. However, directly inputting this data into the network does not yield good results (in fact, the outcomes are quite poor), and I suspect it might be due to calibration issues.

Could you give me some advice or share any calibration code that could help improve my work? Specifically, I am interested in how to preprocess data from VR to make it similar to the input format used in the AMASS dataset. Below, I have included parts of my data along with my preprocessing code for your reference.

Thank you very much for your time and assistance.

Best regards,

Yinghao
`if name == 'main':

# h2r  0:head 1:right_hand 2:left_hand  3:tracker_1 Root  tracker_2:left_foot  tracker_3:right_foot   #linger
dst = 'D:\code\data\VR_6F_H2R\process'
for npz_file in npz_files:
    file_path = os.path.join(cfg.data_path, npz_file)
    data_all = dict()
    data = np.load(file_path)['pose']
    # data_all.append(data['pose'])
    seq = data.shape[0]
    feature_bz = 6
    #head
    hmd_trans = torch.from_numpy((data[:, :3]))
    hmd_rot = torch.from_numpy((np.radians(data[:, 3:6])))
    #right_hand
    right_trans = torch.from_numpy((data[:, 6:9]))
    right_rot = torch.from_numpy((np.radians(data[:, 9:12])))
    #left_hand
    left_trans = torch.from_numpy((data[:, 12:15]))
    left_rot = torch.from_numpy((np.radians(data[:, 15:18])))
    #root
    root_trans = torch.from_numpy((data[:, 18:21]))
    root_rot = torch.from_numpy((np.radians(data[:, 21:24])))
    # left_foot
    left_trans_foot = torch.from_numpy((data[:, 24:27]))
    left_rot_foot = torch.from_numpy((np.radians(data[:, 27:30])))
    # right_foot
    right_trans_foot = torch.from_numpy((data[:, 30:33]))
    right_rot_foot = torch.from_numpy((np.radians(data[:, 33:36])))

    trans_mat = torch.from_numpy(
        np.concatenate((hmd_trans, right_trans, left_trans, root_trans, right_trans_foot, left_trans_foot),
                       axis=1).reshape(-1, feature_bz, 3)).float()
    rot = torch.from_numpy(
        np.concatenate((hmd_rot, right_rot, left_rot, root_rot, right_rot_foot, left_rot_foot),
                       axis=1).reshape(-1,  feature_bz, 3)).float()

    rot_mat = aa2matrot(rot.reshape(-1, 3)).reshape(seq, -1, 3, 3)


    rot_matrot = rot_mat
    trans = trans_mat


    # rot_matrot = aa2matrot(rot.reshape(-1,3)).reshape(seq,-1,9)
    rotation_global_6d = utils_transform.matrot2sixd(rot_matrot.reshape(-1, 3, 3)).reshape(rot_matrot.shape[0], -1, 6)
    rotation_velocity_global_matrot = torch.matmul(torch.inverse(rot_matrot.reshape(-1, feature_bz, 3, 3)[:-1]), rot_matrot.reshape(-1, feature_bz, 3, 3)[1:])
    rotation_velocity_global_6d = utils_transform.matrot2sixd(
        rotation_velocity_global_matrot.reshape(-1, 3, 3)).reshape(rotation_velocity_global_matrot.shape[0], -1, 6)
    data_trans = torch.zeros(seq - 1, 22, 3)
    data_vel = torch.zeros(seq -1, 22, 3)
    data_rot = torch.zeros(seq -1, 22, 6)
    data_rvel = torch.zeros(seq -1, 22, 6)

    indices = np.array([15, 21, 20, 0, 8, 7])
    target = np.array([0, 1, 2, 3, 4, 5])
    data_trans[:, indices, :] = trans[1:, target, :]
    data_vel[:, indices, :] = trans[1:, target, :] - trans[:-1, target, :]
    data_rot[:, indices, :] = rotation_global_6d[1:, target, :]
    data_rvel[:, indices, :] = rotation_velocity_global_6d[:, target]

    hmd_position_global_full_gt_list = torch.cat([data_rot.reshape(seq - 1, -1),
                                                 data_rvel.reshape(seq - 1, -1),
                                                 data_trans.reshape(seq - 1, -1),
                                                 data_vel.reshape(seq - 1, -1)], dim=-1)

    position_head_world = trans[:, 0, ...] # world position of head
    head_global_trans = torch.eye(4).repeat(position_head_world.shape[0], 1, 1)
    head_global_trans[:, :3, :3] = rot_matrot[:, 0, ...].reshape(-1, 3, 3).squeeze()
    head_global_trans[:, :3, 3] = position_head_world
    head_global_trans_list = head_global_trans[1:]


    data_all['hmd_position_global_full_gt_list'] = hmd_position_global_full_gt_list.cpu()
    data_all['head_global_trans_list'] = head_global_trans_list
    data_all['rotation_local_full_gt_list'] = trans
    data_all['body_parms_list'] = rot
    data_all['filepath'] = file_path

    os.makedirs(dst, exist_ok=True)
    idx = npz_file.split('.')[0]
    with open(os.path.join(dst, '{}.pkl'.format(idx)), 'wb') as f:
        pickle.dump(data_all, f)

1.zip

`